From 3a32907cbd0e4d6324ca15341c1d20fea5d61fe9 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 21 Aug 2019 19:17:00 -0700
Subject: [PATCH 01/47] Draft EventProcessor Loadbalancing

---
 .../eventprocessor/_ownership_manager.py      | 120 ++++++++++
 .../eventprocessor/event_processor.py         | 207 +++++++++---------
 .../eventprocessor/partition_processor.py     |  15 +-
 .../sqlite3_partition_manager.py              |  95 ++++----
 4 files changed, 287 insertions(+), 150 deletions(-)
 create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
new file mode 100644
index 000000000000..710e71ab8439
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -0,0 +1,120 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# -----------------------------------------------------------------------------------
+
+from typing import List, Iterable, Any, Dict
+import time
+import random
+import math
+from collections import Counter
+from azure.eventhub.aio import EventHubClient
+
+
+class OwnershipManager(object):
+    """Increases or decreases the number of partitions owned by an EventProcessor
+    so the number of owned partitions are balanced among multiple EventProcessors
+
+    An EventProcessor calls claim_ownership() of this class every x seconds,
+    where x is set by keyword argument "polling_interval" in EventProcessor,
+    to claim the ownership of partitions, create tasks for the claimed ownership, and cancel tasks that no longer belong
+    to the claimed ownership.
+
+    """
+    def __init__(self, event_processor, eventhub_client: EventHubClient, ownership_timeout: int):
+        self.all_parition_ids = []
+        self.eventhub_client = eventhub_client
+        self.eventhub_name = eventhub_client.eh_name
+        self.consumer_group_name = event_processor._consumer_group_name
+        self.owner_id = event_processor._id
+        self.partition_manager = event_processor._partition_manager
+        self.ownership_timeout = ownership_timeout
+
+    async def claim_ownership(self):
+        """Claims ownership for this EventProcessor
+        1. Retrieves all partition ids of an event hub from azure event hub service
+        2. Retrieves current ownership list via this EventProcessor's PartitionManager.
+        3. Searches claimable partitions for this EventProcessor. Refer to claim_ownership() for details.
+        4. Claims the ownership for the claimable partitions
+
+        :return: List[Dict[Any]]
+        """
+        if not self.all_parition_ids:
+            await self._retrieve_partition_ids()
+        to_claim = await self._balance_ownership()
+        claimed_list = await self._claim_ownership(to_claim)
+        return claimed_list
+
+    async def _retrieve_partition_ids(self):
+        """List all partition ids of the event hub that the EventProcessor is working on.
+
+        :return: List[str]
+        """
+        self.all_parition_ids = await self.eventhub_client.get_partition_ids()
+
+    async def _balance_ownership(self):
+        ownership_list = await self.partition_manager.list_ownership(self.eventhub_client.eh_name, self.consumer_group_name)
+        ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
+        '''
+        now = time.time()
+        partition_ids_no_ownership = list(filter(lambda x: x not in ownership_dict, self.all_parition_ids))
+        inactive_ownership = filter(lambda x: x["last_modified_time"] + self.ownership_timeout < now, ownership_list)
+        claimable_partition_ids = partition_ids_no_ownership + [x["partition_id"] for x in inactive_ownership]
+        active_ownership = list(filter(lambda x: x["last_modified_time"] + self.ownership_timeout >= now, ownership_list))
+        active_ownership_count_group_by_owner = Counter([x["owner_id"] for x in active_ownership])
+        active_ownership_self = list(filter(lambda x: x["owner_id"] == self.owner_id, active_ownership))
+        '''
+        claimable_partition_ids = []
+        active_ownership_self = []
+        active_ownership_count_group_by_owner = Counter()
+        for partition_id in self.all_parition_ids:
+            ownership = ownership_dict.get(partition_id)
+            if not ownership:  # no ownership found for this partition. So it is claimable
+                claimable_partition_ids.append(partition_id)
+            else:
+                last_modified_time = ownership["last_modified_time"]
+                owner_id = ownership["owner_id"]
+                now = time.time()
+                if now > self.ownership_timeout + last_modified_time:  # ownership timed out. So it is claimable
+                    claimable_partition_ids.append(partition_id)
+                else:  # the ownership is still active
+                    if owner_id == self.owner_id:  # partition is actively owned by this running EventProcessor
+                        active_ownership_self.append(ownership)
+                    active_ownership_count_group_by_owner[owner_id] = active_ownership_count_group_by_owner.get(owner_id, 0) + 1  # all active owners
+
+        # calculate expected count per owner
+        all_partition_count = len(self.all_parition_ids)
+        owners_count = len(active_ownership_count_group_by_owner) + (1 if self.owner_id not in active_ownership_count_group_by_owner else 0)
+        expected_count_per_owner = all_partition_count // owners_count
+        most_count_allowed_per_owner = math.ceil(all_partition_count / owners_count)
+        # end of calculating expected count per owner
+
+        to_claim = active_ownership_self
+        if len(active_ownership_self) > most_count_allowed_per_owner:  # needs to abandon a partition
+            to_claim.pop()  # abandon one partition if owned too many
+            # TODO: Release a ownership immediately so other EventProcessors won't need to wait it to timeout
+        elif len(active_ownership_self) < expected_count_per_owner:  # Either claims an inactive partition, or steals from other owners
+            if claimable_partition_ids:  # claim an inactive partition if there is
+                random_partition_id = random.choice(claimable_partition_ids)
+                random_chosen_to_claim = ownership_dict.get(random_partition_id,
+                                                            {"partition_id": random_partition_id,
+                                                             "eventhub_name": self.eventhub_client.eh_name,
+                                                             "consumer_group_name": self.consumer_group_name,
+                                                             "owner_level": 0})
+                random_chosen_to_claim["owner_id"] = self.owner_id
+                to_claim.append(random_chosen_to_claim)
+            else:  # steal from another owner that has the most count
+                most_frequent_owner_id = active_ownership_count_group_by_owner.most_common(1)[0][0]
+                # randomly choose a partition to steal from the most_frequent_owner
+                to_steal_partition = random.choice(list(filter(lambda x: x["owner_id"] == most_frequent_owner_id,
+                                                   ownership_list)))
+                to_steal_partition["owner_id"] = self.owner_id
+                to_claim.append(to_steal_partition)
+        return to_claim
+
+    async def _claim_ownership(self, ownership_list):
+        if ownership_list:
+            claimed_list = await self.partition_manager.claim_ownership(ownership_list)
+            return claimed_list
+        else:
+            return None
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 85020257df46..c675afd165a4 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -3,15 +3,17 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # -----------------------------------------------------------------------------------
 
-from typing import Callable, List
+from typing import Callable, Dict
 import uuid
 import asyncio
 import logging
+from enum import Enum
 
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
 from .checkpoint_manager import CheckpointManager
 from .partition_manager import PartitionManager
+from ._ownership_manager import OwnershipManager
 from .partition_processor import PartitionProcessor, CloseReason
 from .utils import get_running_loop
 
@@ -53,7 +55,7 @@ async def process_events(self, events):
 
     """
     def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
-                 partition_processor_factory: Callable[[CheckpointManager], PartitionProcessor],
+                 partition_processor_factory,
                  partition_manager: PartitionManager, **kwargs):
         """
         Instantiate an EventProcessor.
@@ -73,16 +75,22 @@ def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
         :type initial_event_position: int or str
 
         """
+
         self._consumer_group_name = consumer_group_name
         self._eventhub_client = eventhub_client
         self._eventhub_name = eventhub_client.eh_name
         self._partition_processor_factory = partition_processor_factory
         self._partition_manager = partition_manager
         self._initial_event_position = kwargs.get("initial_event_position", "-1")
+        # TODO: initial position provider will be a callable
+        #  so users can create initial event position for every partition
         self._max_batch_size = eventhub_client.config.max_batch_size
         self._receive_timeout = eventhub_client.config.receive_timeout
-        self._tasks = []  # type: List[asyncio.Task]
+        self._polling_interval = kwargs.get("polling_interval", 10)
+        self._ownership_timeout = self._polling_interval * 2  # TODO: Team haven't decided if this is a separate argument
+        self._tasks = {}  # type: Dict[str, asyncio.Task]
         self._id = str(uuid.uuid4())
+        self._running = False
 
     def __repr__(self):
         return 'EventProcessor: id {}'.format(self._id)
@@ -98,9 +106,22 @@ async def start(self):
 
         """
         log.info("EventProcessor %r is being started", self._id)
-        partition_ids = await self._eventhub_client.get_partition_ids()
-        claimed_list = await self._claim_partitions(partition_ids)
-        await self._start_claimed_partitions(claimed_list)
+        ownership_manager = OwnershipManager(self, self._eventhub_client, self._ownership_timeout)
+        if not self._running:
+            self._running = True
+            while self._running:
+                claimed_ownership_list = await ownership_manager.claim_ownership()
+                claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
+                to_cancel_list = self._tasks.keys() - claimed_partition_ids
+                if to_cancel_list:
+                    self._cancel_tasks_for_partitions(to_cancel_list)
+                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
+
+                if claimed_partition_ids:
+                    self._create_tasks_for_claimed_ownership(claimed_ownership_list)
+                else:
+                    log.warning("EventProcessor %r hasn't claimed an ownership. It keeps claiming.", self._id)
+                await asyncio.sleep(self._polling_interval)
 
     async def stop(self):
         """Stop all the partition consumer
@@ -110,109 +131,85 @@ async def stop(self):
         :return: None
 
         """
+        self._running = False
         for i in range(len(self._tasks)):
-            task = self._tasks.pop()
+            task = self._tasks.popitem()[1]
             task.cancel()
         log.info("EventProcessor %r has been cancelled", self._id)
         await asyncio.sleep(2)  # give some time to finish after cancelled
 
-    async def _claim_partitions(self, partition_ids):
-        partitions_ownership = await self._partition_manager.list_ownership(self._eventhub_name, self._consumer_group_name)
-        partitions_ownership_dict = dict()
-        for ownership in partitions_ownership:
-            partitions_ownership_dict[ownership["partition_id"]] = ownership
-
-        to_claim_list = []
-        for pid in partition_ids:
-            p_ownership = partitions_ownership_dict.get(pid)
-            if p_ownership:
-                to_claim_list.append(p_ownership)
-            else:
-                new_ownership = {"eventhub_name": self._eventhub_name, "consumer_group_name": self._consumer_group_name,
-                                 "owner_id": self._id, "partition_id": pid, "owner_level": OWNER_LEVEL}
-                to_claim_list.append(new_ownership)
-        claimed_list = await self._partition_manager.claim_ownership(to_claim_list)
-        return claimed_list
-
-    async def _start_claimed_partitions(self, claimed_partitions):
-        for partition in claimed_partitions:
-            partition_id = partition["partition_id"]
-            offset = partition.get("offset", self._initial_event_position)
-            consumer = self._eventhub_client.create_consumer(self._consumer_group_name, partition_id,
-                                                            EventPosition(str(offset)))
-            partition_processor = self._partition_processor_factory(
-                checkpoint_manager=CheckpointManager(partition_id, self._eventhub_name, self._consumer_group_name,
-                                                     self._id, self._partition_manager)
-            )
-            loop = get_running_loop()
-            task = loop.create_task(
-                _receive(consumer, partition_processor, self._receive_timeout))
-            self._tasks.append(task)
+    def _cancel_tasks_for_partitions(self, to_cancel_partitions):
+        for partition_id in to_cancel_partitions:
+            if partition_id in self._tasks:
+                task = self._tasks.pop(partition_id)
+                task.cancel()
+
+    def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list):
+        for ownership in to_claim_ownership_list:
+            partition_id = ownership["partition_id"]
+            if partition_id not in self._tasks:
+                self._tasks[partition_id] = asyncio.create_task(self._receive(ownership))
+
+    async def _receive(self, ownership):
+        log.info("start ownership, %r", ownership)
+        partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
+                                                                   ownership["partition_id"],
+                                                                   EventPosition(ownership.get("offset", self._initial_event_position))
+                                                                   )
+        checkpoint_manager = CheckpointManager(ownership["partition_id"],
+                                               ownership["eventhub_name"],
+                                               ownership["consumer_group_name"],
+                                               ownership["owner_id"],
+                                               self._partition_manager)
+        partition_processor = self._partition_processor_factory()
+
+        async def initialize():
+            if hasattr(partition_processor, "initialize"):
+                await partition_processor.initialize(checkpoint_manager)
+
+        async def process_error(err):
+            if hasattr(partition_processor, "process_error"):
+                await partition_processor.process_error(err, checkpoint_manager)
+
+        async def close(close_reason):
+            if hasattr(partition_processor, "close"):
+                await partition_processor.close(close_reason, checkpoint_manager)
+
         try:
-            await asyncio.gather(*self._tasks)
+            while True:
+                try:
+                    await initialize()
+                    events = await partition_consumer.receive(timeout=self._receive_timeout)
+                    await partition_processor.process_events(events, checkpoint_manager)
+                except asyncio.CancelledError as cancelled_error:
+                    log.info(
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
+                        "is cancelled",
+                        ownership["owner_id"],
+                        ownership["eventhub_name"],
+                        ownership["partition_id"],
+                        ownership["consumer_group_name"]
+                    )
+                    await process_error(cancelled_error)
+                    await close(CloseReason.SHUTDOWN)
+                    break
+                except EventHubError as eh_err:
+                    reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
+                    log.warning(
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
+                        "has met an exception receiving events. It's being closed. The exception is %r.",
+                        ownership["owner_id"],
+                        ownership["eventhub_name"],
+                        ownership["partition_id"],
+                        ownership["consumer_group_name"],
+                        eh_err
+                    )
+                    await process_error(eh_err)
+                    await close(reason)
+                    break
+                except Exception as exp:
+                    log.warning(exp)
+                    # TODO: will review whether to break and close partition processor after user's code has an exception
+            # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
         finally:
-            log.info("EventProcessor %r has stopped", self._id)
-
-
-async def _receive(partition_consumer, partition_processor, receive_timeout):
-    try:
-        while True:
-            try:
-                events = await partition_consumer.receive(timeout=receive_timeout)
-            except asyncio.CancelledError as cancelled_error:
-                log.info(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "is cancelled",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name
-                )
-                await partition_processor.process_error(cancelled_error)
-                await partition_processor.close(reason=CloseReason.SHUTDOWN)
-                break
-            except EventHubError as eh_err:
-                reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
-                log.warning(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "has met an exception receiving events. It's being closed. The exception is %r.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name,
-                    eh_err
-                )
-                await partition_processor.process_error(eh_err)
-                await partition_processor.close(reason=reason)
-                break
-            try:
-                await partition_processor.process_events(events)
-            except asyncio.CancelledError as cancelled_error:
-                log.info(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "is cancelled.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name
-                )
-                await partition_processor.process_error(cancelled_error)
-                await partition_processor.close(reason=CloseReason.SHUTDOWN)
-                break
-            except Exception as exp:  # user code has caused an error
-                log.warning(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "has met an exception from user code process_events. It's being closed. The exception is %r.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name,
-                    exp
-                )
-                await partition_processor.process_error(exp)
-                await partition_processor.close(reason=CloseReason.EVENTHUB_EXCEPTION)
-                break
-                # TODO: will review whether to break and close partition processor after user's code has an exception
-        # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
-    finally:
-        await partition_consumer.close()
+            await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 10aafc79c492..94729aae1269 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -4,7 +4,7 @@
 # -----------------------------------------------------------------------------------
 
 from typing import List
-from abc import ABC, abstractmethod
+from typing_extensions import Protocol
 from enum import Enum
 from .checkpoint_manager import CheckpointManager
 
@@ -17,16 +17,16 @@ class CloseReason(Enum):
     EVENTHUB_EXCEPTION = 2  # Exception happens during receiving events
 
 
-class PartitionProcessor(ABC):
+class PartitionProcessor(Protocol):
     """
     PartitionProcessor processes events received from the Azure Event Hubs service. A single instance of a class
     implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns.
 
     """
-    def __init__(self, checkpoint_manager: CheckpointManager):
-        self._checkpoint_manager = checkpoint_manager
+    async def initialize(self, checkpoint_manager: CheckpointManager):
+        pass
 
-    async def close(self, reason):
+    async def close(self, reason, checkpoint_manager: CheckpointManager):
         """Called when EventProcessor stops processing this PartitionProcessor.
 
         There are different reasons to trigger the PartitionProcessor to close.
@@ -38,8 +38,7 @@ async def close(self, reason):
         """
         pass
 
-    @abstractmethod
-    async def process_events(self, events: List[EventData]):
+    async def process_events(self, events: List[EventData], checkpoint_manager: CheckpointManager):
         """Called when a batch of events have been received.
 
         :param events: Received events.
@@ -48,7 +47,7 @@ async def process_events(self, events: List[EventData]):
         """
         pass
 
-    async def process_error(self, error):
+    async def process_error(self, error, checkpoint_manager: CheckpointManager):
         """Called when an error happens
 
         :param error: The error that happens.
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index eb08e970fa89..5e2c3dd45748 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -6,8 +6,11 @@
 import time
 import uuid
 import sqlite3
+import logging
 from .partition_manager import PartitionManager
 
+logger = logging.getLogger(__name__)
+
 
 def _check_table_name(table_name: str):
     for c in table_name:
@@ -22,6 +25,15 @@ class Sqlite3PartitionManager(PartitionManager):
 
 
     """
+    primary_keys_dict = {"eventhub_name": "text", "consumer_group_name": "text", "partition_id": "text"}
+    other_fields_dict = {"owner_id": "text", "owner_level": "integer", "sequence_number": "integer", "offset": "text",
+                         "last_modified_time": "real", "etag": "text"}
+    checkpoint_fields = ["sequence_number", "offset"]
+    fields_dict = {**primary_keys_dict, **other_fields_dict}
+    primary_keys = list(primary_keys_dict.keys())
+    other_fields = list(other_fields_dict.keys())
+    fields = primary_keys + other_fields
+
     def __init__(self, db_filename: str = ":memory:", ownership_table: str = "ownership"):
         """
 
@@ -34,17 +46,15 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
         conn = sqlite3.connect(db_filename)
         c = conn.cursor()
         try:
-            c.execute("create table " + ownership_table +
-                      "(eventhub_name text,"
-                      "consumer_group_name text,"
-                      "owner_id text,"
-                      "partition_id text,"
-                      "owner_level integer,"
-                      "sequence_number integer,"
-                      "offset text,"
-                      "last_modified_time integer,"
-                      "etag text)")
+            sql = "create table if not exists " + _check_table_name(ownership_table)\
+                  + "("\
+                  + ",".join([x[0]+" "+x[1] for x in self.fields_dict.items()])\
+                  + ", constraint pk_ownership PRIMARY KEY ("\
+                  + ",".join(self.primary_keys)\
+                  + "))"
+            c.execute(sql)
         except sqlite3.OperationalError:
+            raise
             pass
         finally:
             c.close()
@@ -53,44 +63,55 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
     async def list_ownership(self, eventhub_name, consumer_group_name):
         cursor = self.conn.cursor()
         try:
-            fields = ["eventhub_name", "consumer_group_name", "owner_id", "partition_id", "owner_level",
-                      "sequence_number",
-                      "offset", "last_modified_time", "etag"]
-            cursor.execute("select " + ",".join(fields) +
+            cursor.execute("select " + ",".join(self.fields) +
                                 " from "+_check_table_name(self.ownership_table)+" where eventhub_name=? "
                                 "and consumer_group_name=?",
                                 (eventhub_name, consumer_group_name))
-            result_list = []
-
-            for row in cursor.fetchall():
-                d = dict(zip(fields, row))
-                result_list.append(d)
-            return result_list
+            return [dict(zip(self.fields, row)) for row in cursor.fetchall()]
         finally:
             cursor.close()
 
     async def claim_ownership(self, partitions):
+        result = []
         cursor = self.conn.cursor()
         try:
             for p in partitions:
-                cursor.execute("select * from " + _check_table_name(self.ownership_table) +
-                                    " where eventhub_name=? "
-                                    "and consumer_group_name=? "
-                                    "and partition_id =?",
-                                    (p["eventhub_name"], p["consumer_group_name"],
-                                     p["partition_id"]))
-                if not cursor.fetchall():
-                    cursor.execute("insert into " + _check_table_name(self.ownership_table) +
-                                   " (eventhub_name,consumer_group_name,partition_id,owner_id,owner_level,last_modified_time,etag) "
-                                   "values (?,?,?,?,?,?,?)",
-                                   (p["eventhub_name"], p["consumer_group_name"], p["partition_id"], p["owner_id"], p["owner_level"],
-                                    time.time(), str(uuid.uuid4())
-                                    ))
+                cursor.execute("select etag from " + _check_table_name(self.ownership_table) +
+                                    " where "+ " and ".join([field+"=?" for field in self.primary_keys]),
+                                    tuple(p.get(field) for field in self.primary_keys))
+                cursor_fetch = cursor.fetchall()
+                if not cursor_fetch:
+                    p["last_modified_time"] = time.time()
+                    p["etag"] = str(uuid.uuid4())
+                    try:
+                        fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.fields))
+                        sql = "insert into " + _check_table_name(self.ownership_table) + " (" \
+                              + ",".join(fields_without_checkpoint) \
+                              + ") values (?,?,?,?,?,?,?)"
+                        cursor.execute(sql, tuple(p.get(field) for field in fields_without_checkpoint))
+                    except sqlite3.OperationalError as op_err:
+                        logger.info("EventProcessor %r failed to claim partition %r "
+                                    "because it was claimed by another EventProcessor at the same time. "
+                                    "The Sqlite3 exception is %r", p["owner_id"], p["partition_id"], op_err)
+                        break
+                    else:
+                        result.append(p)
                 else:
-                    cursor.execute("update " + _check_table_name(self.ownership_table) + " set owner_id=?, owner_level=?, last_modified_time=?, etag=? "
-                                   "where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                                   (p["owner_id"], p["owner_level"], time.time(), str(uuid.uuid4()),
-                                    p["eventhub_name"], p["consumer_group_name"], p["partition_id"]))
+                    if p.get("etag") == cursor_fetch[0][0]:
+                        p["last_modified_time"] = time.time()
+                        p["etag"] = str(uuid.uuid4())
+                        other_fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.other_fields))
+                        sql = "update " + _check_table_name(self.ownership_table) + " set "\
+                                       + ','.join([field+"=?" for field in other_fields_without_checkpoint])\
+                                       + " where "\
+                                       + " and ".join([field+"=?" for field in self.primary_keys])
+
+                        cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint) + tuple(p.get(field) for field in self.primary_keys))
+                        result.append(p)
+                    else:
+                        logger.info("EventProcessor %r failed to claim partition %r "
+                                    "because it was claimed by another EventProcessor at the same time", p["owner_id"],
+                                    p["partition_id"])
             self.conn.commit()
             return partitions
         finally:

From 39b1b86853f5f4a4ca52afc80ef6a20e8fe51615 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 01:01:35 -0700
Subject: [PATCH 02/47] EventProcessor Load balancing

---
 .../eventprocessor/_ownership_manager.py      | 45 ++++++++-----
 .../eventprocessor/event_processor.py         | 66 +++++++++++--------
 .../eventprocessor/partition_processor.py     |  4 +-
 .../sqlite3_partition_manager.py              | 13 +++-
 .../eventprocessor/event_processor_example.py | 39 ++++-------
 5 files changed, 94 insertions(+), 73 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
index 710e71ab8439..40e3419a7c5a 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # -----------------------------------------------------------------------------------
 
-from typing import List, Iterable, Any, Dict
 import time
 import random
 import math
@@ -34,8 +33,8 @@ async def claim_ownership(self):
         """Claims ownership for this EventProcessor
         1. Retrieves all partition ids of an event hub from azure event hub service
         2. Retrieves current ownership list via this EventProcessor's PartitionManager.
-        3. Searches claimable partitions for this EventProcessor. Refer to claim_ownership() for details.
-        4. Claims the ownership for the claimable partitions
+        3. Balances number of ownership. Refer to _balance_ownership() for details.
+        4. Claims the ownership for the balanced number of partitions.
 
         :return: List[Dict[Any]]
         """
@@ -53,19 +52,35 @@ async def _retrieve_partition_ids(self):
         self.all_parition_ids = await self.eventhub_client.get_partition_ids()
 
     async def _balance_ownership(self):
+        """Balances and claims ownership of partitions for this EventProcessor.
+        The balancing algorithm is:
+        1. Find partitions with inactive ownership and partitions that haven never been claimed before
+        2. Find the number of active owners, including this EventProcessor, for all partitions.
+        3. Calculate the average count of partitions that an owner should own.
+        (number of partitions // number of active owners)
+        4. Calculate the largest allowed count of partitions that an owner can own.
+        math.ceil(number of partitions / number of active owners). This should be equal or 1 greater than the average count
+        5. Adjust the number of partitions owned by this EventProcessor (owner)
+            a. if this EventProcessor owns more than largest allowed count, abandon one partition
+            b. if this EventProcessor owns less than average count, add one from the inactive or unclaimed partitions,
+                or steal one from another owner that has the largest number of ownership among all owners (EventProcessors)
+            c. Otherwise, no change to the ownership
+
+        The balancing algorithm adjust one partition at a time to gradually build the balanced ownership.
+        Ownership must be renewed to keep it active. So the returned result includes both existing ownership and
+        the newly adjusted ownership.
+        This method balances but doesn't claim ownership. The caller of this method tries to claim the result ownership
+        list. But it may not successfully claim all of them because of concurrency. Other EventProcessors may happen to
+        claim a partition at that time. Since balancing and claiming are run in infinite repeatedly,
+        it achieves balancing among all EventProcessors after some time of running.
+
+        :return: List[Dict[str, Any]], A list of ownership.
+        """
         ownership_list = await self.partition_manager.list_ownership(self.eventhub_client.eh_name, self.consumer_group_name)
         ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
-        '''
-        now = time.time()
-        partition_ids_no_ownership = list(filter(lambda x: x not in ownership_dict, self.all_parition_ids))
-        inactive_ownership = filter(lambda x: x["last_modified_time"] + self.ownership_timeout < now, ownership_list)
-        claimable_partition_ids = partition_ids_no_ownership + [x["partition_id"] for x in inactive_ownership]
-        active_ownership = list(filter(lambda x: x["last_modified_time"] + self.ownership_timeout >= now, ownership_list))
-        active_ownership_count_group_by_owner = Counter([x["owner_id"] for x in active_ownership])
-        active_ownership_self = list(filter(lambda x: x["owner_id"] == self.owner_id, active_ownership))
-        '''
-        claimable_partition_ids = []
-        active_ownership_self = []
+
+        claimable_partition_ids = []  # partitions with inactive ownership and partitions that have never been claimed yet
+        active_ownership_self = []  # active ownership of this EventProcessor
         active_ownership_count_group_by_owner = Counter()
         for partition_id in self.all_parition_ids:
             ownership = ownership_dict.get(partition_id)
@@ -100,7 +115,7 @@ async def _balance_ownership(self):
                                                             {"partition_id": random_partition_id,
                                                              "eventhub_name": self.eventhub_client.eh_name,
                                                              "consumer_group_name": self.consumer_group_name,
-                                                             "owner_level": 0})
+                                                             "owner_level": 0})  # TODO: consider removing owner_level
                 random_chosen_to_claim["owner_id"] = self.owner_id
                 to_claim.append(random_chosen_to_claim)
             else:  # steal from another owner that has the most count
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index c675afd165a4..7e342a369d0d 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -7,15 +7,13 @@
 import uuid
 import asyncio
 import logging
-from enum import Enum
 
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
 from .checkpoint_manager import CheckpointManager
 from .partition_manager import PartitionManager
 from ._ownership_manager import OwnershipManager
-from .partition_processor import PartitionProcessor, CloseReason
-from .utils import get_running_loop
+from .partition_processor import CloseReason, PartitionProcessor
 
 log = logging.getLogger(__name__)
 
@@ -29,20 +27,21 @@ class EventProcessor(object):
 
     It provides the user a convenient way to receive events from multiple partitions and save checkpoints.
     If multiple EventProcessors are running for an event hub, they will automatically balance load.
-    This load balancing won't be available until preview 3.
 
     Example:
         .. code-block:: python
 
-            class MyPartitionProcessor(PartitionProcessor):
-                async def process_events(self, events):
-                    if events:
-                        # do something sync or async to process the events
-                        await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
-
             import asyncio
             from azure.eventhub.aio import EventHubClient
             from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager
+
+            class MyPartitionProcessor(object):
+                async def process_events(self, events, checkpoint_manager):
+                    if events:
+                        # do something sync or async to process the events
+                        await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+
+
             client = EventHubClient.from_connection_string("<your connection string>", receive_timeout=5, retry_total=3)
             partition_manager = Sqlite3PartitionManager()
             try:
@@ -55,7 +54,7 @@ async def process_events(self, events):
 
     """
     def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
-                 partition_processor_factory,
+                 partition_processor_factory: Callable[..., PartitionProcessor],
                  partition_manager: PartitionManager, **kwargs):
         """
         Instantiate an EventProcessor.
@@ -73,6 +72,8 @@ def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
         :type partition_manager: Class implementing the ~azure.eventhub.eventprocessor.PartitionManager.
         :param initial_event_position: The offset to start a partition consumer if the partition has no checkpoint yet.
         :type initial_event_position: int or str
+        :param polling_interval: The interval between any two pollings of balancing and claiming
+        :type float
 
         """
 
@@ -98,9 +99,12 @@ def __repr__(self):
     async def start(self):
         """Start the EventProcessor.
 
-        1. retrieve the partition ids from eventhubs.
-        2. claim partition ownership of these partitions.
-        3. repeatedly call EvenHubConsumer.receive() to retrieve events and call user defined PartitionProcessor.process_events().
+        1. Calls the OwnershipManager to keep claiming and balancing ownership of partitions in an
+        infinitely loop until self.stop() is called.
+        2. Cancels tasks for partitions that are no longer owned by this EventProcessor
+        3. Creates tasks for partitions that are newly claimed by this EventProcessor
+        4. Keeps tasks running for partitions that haven't changed ownership
+        5. Each task repeatedly calls EvenHubConsumer.receive() to retrieve events and call user defined partition processor
 
         :return: None
 
@@ -111,22 +115,23 @@ async def start(self):
             self._running = True
             while self._running:
                 claimed_ownership_list = await ownership_manager.claim_ownership()
-                claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
-                to_cancel_list = self._tasks.keys() - claimed_partition_ids
-                if to_cancel_list:
-                    self._cancel_tasks_for_partitions(to_cancel_list)
-                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
-
-                if claimed_partition_ids:
+                if claimed_ownership_list:
+                    claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
+                    to_cancel_list = self._tasks.keys() - claimed_partition_ids
                     self._create_tasks_for_claimed_ownership(claimed_ownership_list)
                 else:
                     log.warning("EventProcessor %r hasn't claimed an ownership. It keeps claiming.", self._id)
+                    to_cancel_list = self._tasks.keys()
+                if to_cancel_list:
+                    self._cancel_tasks_for_partitions(to_cancel_list)
+                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
                 await asyncio.sleep(self._polling_interval)
 
     async def stop(self):
-        """Stop all the partition consumer
+        """Stop claiming ownership and all the partition consumers owned by this EventProcessor
 
-        This method cancels tasks that are running EventHubConsumer.receive() for the partitions owned by this EventProcessor.
+        This method stops claiming ownership of owned partitions and cancels tasks that are running
+        EventHubConsumer.receive() for the partitions owned by this EventProcessor.
 
         :return: None
 
@@ -152,6 +157,13 @@ def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list):
 
     async def _receive(self, ownership):
         log.info("start ownership, %r", ownership)
+        partition_processor = self._partition_processor_factory()
+        if not hasattr(partition_processor, "process_events"):
+            log.error(
+                "Fatal error: a partition processor should at least have method process_events(events, checkpoint_manager). EventProcessor will stop.")
+            await self.stop()
+            raise TypeError("Partition processor must has method process_events(events, checkpoint_manager")
+
         partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
                                                                    ownership["partition_id"],
                                                                    EventPosition(ownership.get("offset", self._initial_event_position))
@@ -161,8 +173,6 @@ async def _receive(self, ownership):
                                                ownership["consumer_group_name"],
                                                ownership["owner_id"],
                                                self._partition_manager)
-        partition_processor = self._partition_processor_factory()
-
         async def initialize():
             if hasattr(partition_processor, "initialize"):
                 await partition_processor.initialize(checkpoint_manager)
@@ -192,6 +202,7 @@ async def close(close_reason):
                     )
                     await process_error(cancelled_error)
                     await close(CloseReason.SHUTDOWN)
+                    # TODO: release the ownership immediately via partition manager
                     break
                 except EventHubError as eh_err:
                     reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
@@ -205,11 +216,10 @@ async def close(close_reason):
                         eh_err
                     )
                     await process_error(eh_err)
-                    await close(reason)
+                    await close(reason)  # An EventProcessor will pick up this partition again after the ownership is released
+                    # TODO: release the ownership immediately via partition manager
                     break
                 except Exception as exp:
                     log.warning(exp)
-                    # TODO: will review whether to break and close partition processor after user's code has an exception
-            # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
         finally:
             await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 94729aae1269..60a95b99e628 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -23,7 +23,7 @@ class PartitionProcessor(Protocol):
     implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns.
 
     """
-    async def initialize(self, checkpoint_manager: CheckpointManager):
+    async def initialize(self):
         pass
 
     async def close(self, reason, checkpoint_manager: CheckpointManager):
@@ -45,7 +45,7 @@ async def process_events(self, events: List[EventData], checkpoint_manager: Chec
         :type events: list[~azure.eventhub.common.EventData]
 
         """
-        pass
+        raise NotImplementedError
 
     async def process_error(self, error, checkpoint_manager: CheckpointManager):
         """Called when an error happens
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index 5e2c3dd45748..bf4c46c95eb9 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -121,9 +121,16 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             offset, sequence_number):
         cursor = self.conn.cursor()
         try:
-            cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                           (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
-            self.conn.commit()
+            cursor.execute("select owner_id from " + _check_table_name(self.ownership_table) + " where eventhub_name=? and consumer_group_name=? and partition_id=?",
+                           (eventhub_name, consumer_group_name, partition_id))
+            cursor_fetch = cursor.fetchall()
+            if cursor_fetch and owner_id == cursor_fetch[0][0]:
+                cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?",
+                               (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
+                self.conn.commit()
+            else:
+                logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership", partition_id)
+
         finally:
             cursor.close()
 
diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
index 8c4c9ced7d29..a2165eb81110 100644
--- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
+++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
@@ -3,7 +3,6 @@
 import os
 from azure.eventhub.aio import EventHubClient
 from azure.eventhub.eventprocessor import EventProcessor
-from azure.eventhub.eventprocessor import PartitionProcessor
 from azure.eventhub.eventprocessor import Sqlite3PartitionManager
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
@@ -18,33 +17,23 @@ async def do_operation(event):
     print(event)
 
 
-class MyPartitionProcessor(PartitionProcessor):
-    def __init__(self, checkpoint_manager):
-        super(MyPartitionProcessor, self).__init__(checkpoint_manager)
-
-    async def process_events(self, events):
+class MyPartitionProcessor(object):
+    async def process_events(self, events, checkpoint_manager):
         if events:
             await asyncio.gather(*[do_operation(event) for event in events])
-            await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
-
-
-def partition_processor_factory(checkpoint_manager):
-    return MyPartitionProcessor(checkpoint_manager)
-
-
-async def run_awhile(duration):
-    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT,
-                                                   retry_total=RETRY_TOTAL)
-    partition_manager = Sqlite3PartitionManager()
-    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager)
-    try:
-        asyncio.ensure_future(event_processor.start())
-        await asyncio.sleep(duration)
-        await event_processor.stop()
-    finally:
-        await partition_manager.close()
+            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+        else:
+            print("empty events received", "partition:", checkpoint_manager.partition_id)
 
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(run_awhile(60))
+    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
+    partition_manager = Sqlite3PartitionManager(db_filename="eventprocessor_test_db")
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
+    try:
+        loop.run_until_complete(event_processor.start())
+    except KeyboardInterrupt:
+        loop.run_until_complete(event_processor.stop())
+    finally:
+        loop.stop()

From 17f5153ed3762667cf5aa8d330dcec9d339512b3 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:10:35 -0700
Subject: [PATCH 03/47] small changes from bryan's review

---
 .../azure/eventhub/eventprocessor/event_processor.py        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 7e342a369d0d..2db6feb4bbf1 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -137,8 +137,8 @@ async def stop(self):
 
         """
         self._running = False
-        for i in range(len(self._tasks)):
-            task = self._tasks.popitem()[1]
+        for _ in range(len(self._tasks)):
+            _, task = self._tasks.popitem()
             task.cancel()
         log.info("EventProcessor %r has been cancelled", self._id)
         await asyncio.sleep(2)  # give some time to finish after cancelled
@@ -205,7 +205,7 @@ async def close(close_reason):
                     # TODO: release the ownership immediately via partition manager
                     break
                 except EventHubError as eh_err:
-                    reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
+                    reason = CloseReason.OWNERSHIP_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
                     log.warning(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
                         "has met an exception receiving events. It's being closed. The exception is %r.",

From 04ef5484e03ac1789e2f9f2e8c6723e85a392c0f Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:21:05 -0700
Subject: [PATCH 04/47] remove checkpoint manager from initialize

---
 .../azure/eventhub/eventprocessor/event_processor.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 2db6feb4bbf1..e808d603c457 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -175,7 +175,7 @@ async def _receive(self, ownership):
                                                self._partition_manager)
         async def initialize():
             if hasattr(partition_processor, "initialize"):
-                await partition_processor.initialize(checkpoint_manager)
+                await partition_processor.initialize()
 
         async def process_error(err):
             if hasattr(partition_processor, "process_error"):

From 9be17413316d4bf556472e950b3bfbeb56df166b Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:41:18 -0700
Subject: [PATCH 05/47] small changes

---
 .../eventprocessor/event_processor.py         | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index e808d603c457..23178deba76a 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -164,15 +164,19 @@ async def _receive(self, ownership):
             await self.stop()
             raise TypeError("Partition processor must has method process_events(events, checkpoint_manager")
 
-        partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
-                                                                   ownership["partition_id"],
-                                                                   EventPosition(ownership.get("offset", self._initial_event_position))
-                                                                   )
-        checkpoint_manager = CheckpointManager(ownership["partition_id"],
-                                               ownership["eventhub_name"],
-                                               ownership["consumer_group_name"],
-                                               ownership["owner_id"],
-                                               self._partition_manager)
+        partition_consumer = self._eventhub_client.create_consumer(
+            ownership["consumer_group_name"],
+            ownership["partition_id"],
+            EventPosition(ownership.get("offset", self._initial_event_position))
+        )
+        checkpoint_manager = CheckpointManager(
+            ownership["partition_id"],
+            ownership["eventhub_name"],
+            ownership["consumer_group_name"],
+            ownership["owner_id"],
+            self._partition_manager
+        )
+
         async def initialize():
             if hasattr(partition_processor, "initialize"):
                 await partition_processor.initialize()

From 875841e63c760b9e84fc903ca53bfea7f58ae503 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 28 Aug 2019 13:42:32 -0700
Subject: [PATCH 06/47] initial blob storage

---
 .../HISTORY.md                                |  0
 .../LICENSE                                   | 21 +++++
 .../MANIFEST.in                               |  2 +
 .../README.md                                 |  0
 .../azure/__init__.py                         |  0
 .../azure/ehplugin/__init__.py                |  0
 .../ehplugin/partitionmanager/__init__.py     |  0
 .../partitionmanager/blobstorage/__init__.py  |  6 ++
 .../blobstorage/blobstoragepm.py              | 80 +++++++++++++++++++
 .../conftest.py                               |  0
 .../dev_requirements.txt                      |  1 +
 .../event_processor_blob_storage_example.py   | 42 ++++++++++
 .../sdk_packaging.toml                        |  2 +
 .../setup.cfg                                 |  2 +
 .../setup.py                                  | 80 +++++++++++++++++++
 15 files changed, 236 insertions(+)
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
new file mode 100644
index 000000000000..21071075c245
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
new file mode 100644
index 000000000000..50c61fef797b
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.md
+include azure/__init__.py
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
new file mode 100644
index 000000000000..3057ad223050
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
@@ -0,0 +1,6 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+__version__ = "1.0.0b1"
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
new file mode 100644
index 000000000000..f0e0230be351
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -0,0 +1,80 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+from typing import Iterable, Dict, Any
+import logging
+from collections import defaultdict
+import asyncio
+
+from azure.eventhub.eventprocessor import PartitionManager
+from azure.storage.blob.aio import ContainerClient
+
+logger = logging.getLogger(__name__)
+UPLOAD_DATA = ""
+
+
+class BlobPartitionManager(PartitionManager):
+    def __init__(self, container_client: ContainerClient):
+        self._container_client = container_client
+        # self._ownership_cache = {}
+        # self._ownership_locks = defaultdict(asyncio.Lock)
+
+    async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
+        blobs = self._container_client.list_blobs(include=['metadata'])
+        result = []
+        async for b in blobs:
+            metadata = b.metadata
+            ownership = {
+                "eventhub_name": eventhub_name,
+                "consumer_group_name": consumer_group_name,
+                "partition_id": b.name,
+                "etag": b.etag,
+                "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
+            }
+            ownership.update(metadata)
+            result.append(ownership)
+        return result
+
+    async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
+        result = []
+        for ownership in ownership_list:
+            metadata = {"owner_id": ownership["owner_id"]}
+            if "offset" in ownership:
+                metadata["offset"] = ownership["offset"]
+            if "sequence_number" in ownership:
+                metadata["sequence_number"] = ownership["sequence_number"]
+            name = ownership["partition_id"]
+            try:
+                etag = ownership.get("etag")
+                if etag:
+                    etag_match = {"if_match": '"'+etag+'"'}
+                else:
+                    etag_match = {"if_none_match": '"*"'}
+                blob_client = await self._container_client.upload_blob(
+                    name=name, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
+                )
+
+                uploaded_blob_properties = await blob_client.get_blob_properties()
+                ownership["etag"] = uploaded_blob_properties.etag
+                ownership["last_modified_time"] = uploaded_blob_properties.last_modified
+            except Exception as err:
+                logger.info("Claim error occurred: %r", err)
+                raise
+            result.append(ownership)
+        return result
+
+    async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
+                                offset, sequence_number) -> None:
+
+        metadata = {
+            "owner_id": owner_id,
+            "offset": offset,
+            "sequence_number": sequence_number
+        }
+        try:
+            blob_client = await self._container_client.upload_blob(name=partition_id, data=UPLOAD_DATA, overwrite=True)
+
+        except Exception as err:
+            logger.info("Checkpoint error occurred: %r", err)
+            raise
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
new file mode 100644
index 000000000000..092dbcdb7de7
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../../eventhub/azure-eventhubs
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
new file mode 100644
index 000000000000..519a3c468460
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -0,0 +1,42 @@
+import asyncio
+import logging
+import os
+from azure.eventhub.aio import EventHubClient
+from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
+from azure.ehplugin.partitionmanager.blobstorage.blobstoragepm import BlobPartitionManager
+from azure.storage.blob.aio import ContainerClient
+
+RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
+RETRY_TOTAL = 3  # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small
+CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
+STORAGE_CONNECTION_STR = "DefaultEndpointsProtocol=https;AccountName=ephstg;AccountKey=JX+/0R9IFtbo406MlvgLb4/ihT66kZh7vyeqUoDZkS5RKeUi7CBO+3mpl1gnNTf2Do6uUFjDQtdr1i2idcQ7Uw==;BlobEndpoint=https://ephstg.blob.core.windows.net/;QueueEndpoint=https://ephstg.queue.core.windows.net/;TableEndpoint=https://ephstg.table.core.windows.net/;FileEndpoint=https://ephstg.file.core.windows.net/;"
+
+logging.basicConfig(level=logging.INFO)
+
+
+async def do_operation(event):
+    # do some sync or async operations. If the operation is i/o intensive, async will have better performance
+    print(event)
+
+
+class MyPartitionProcessor(PartitionProcessor):
+    async def process_events(self, events, checkpoint_manager):
+        if events:
+            await asyncio.gather(*[do_operation(event) for event in events])
+            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+        else:
+            print("empty events received", "partition:", checkpoint_manager.partition_id)
+
+
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
+    container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, container="eventprocessor")
+    partition_manager = BlobPartitionManager(container_client=container_client)
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
+    try:
+        loop.run_until_complete(event_processor.start())
+    except KeyboardInterrupt:
+        loop.run_until_complete(event_processor.stop())
+    finally:
+        loop.stop()
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
new file mode 100644
index 000000000000..e7687fdae93b
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
@@ -0,0 +1,2 @@
+[packaging]
+auto_update = false
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
new file mode 100644
index 000000000000..3480374bc2f2
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
new file mode 100644
index 000000000000..78e798359dfe
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#--------------------------------------------------------------------------
+
+import re
+import os.path
+import sys
+from io import open
+from setuptools import find_packages, setup
+
+
+# Change the PACKAGE_NAME only to change folder and different name
+PACKAGE_NAME = "azure-ehplugin-partitionmanager-blobstorage"
+PACKAGE_PPRINT_NAME = "Event Hubs Event Procesor Partition Manager implementation with Blob Storage"
+
+# a-b-c => a/b/c
+package_folder_path = PACKAGE_NAME.replace('-', '/')
+# a-b-c => a.b.c
+namespace_name = PACKAGE_NAME.replace('-', '.')
+
+# Version extraction inspired from 'requests'
+with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd:
+    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
+                        fd.read(), re.MULTILINE).group(1)
+
+if not version:
+    raise RuntimeError('Cannot find version information')
+
+with open('README.md') as f:
+    readme = f.read()
+with open('HISTORY.md') as f:
+    history = f.read()
+
+exclude_packages = [
+        'tests',
+        'examples',
+        # Exclude packages that will be covered by PEP420 or nspkg
+        'azure',
+    ]
+
+if sys.version_info < (3, 5, 3):
+    exclude_packages.extend([
+        '*.aio',
+        '*.aio.*',
+    ])
+
+setup(
+    name=PACKAGE_NAME,
+    version=version,
+    description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
+    long_description=readme + '\n\n' + history,
+    long_description_content_type='text/markdown',
+    license='MIT License',
+    author='Microsoft Corporation',
+    author_email='azpysdkhelp@microsoft.com',
+    url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'License :: OSI Approved :: MIT License',
+    ],
+    zip_safe=False,
+    packages=find_packages(exclude=exclude_packages),
+    install_requires=[
+
+    ],
+    extras_require={
+        ":python_version<'3.0'": ['azure-nspkg'],
+    }
+)

From 6df72535ad9b1814389348df744d6a3af5e2b9f3 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 28 Aug 2019 14:13:26 -0700
Subject: [PATCH 07/47] fix leaking connection str

---
 .../examples/event_processor_blob_storage_example.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
index 519a3c468460..a802bb2b3a64 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -9,7 +9,7 @@
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
 RETRY_TOTAL = 3  # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small
 CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
-STORAGE_CONNECTION_STR = "DefaultEndpointsProtocol=https;AccountName=ephstg;AccountKey=JX+/0R9IFtbo406MlvgLb4/ihT66kZh7vyeqUoDZkS5RKeUi7CBO+3mpl1gnNTf2Do6uUFjDQtdr1i2idcQ7Uw==;BlobEndpoint=https://ephstg.blob.core.windows.net/;QueueEndpoint=https://ephstg.queue.core.windows.net/;TableEndpoint=https://ephstg.table.core.windows.net/;FileEndpoint=https://ephstg.file.core.windows.net/;"
+STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
 
 logging.basicConfig(level=logging.INFO)
 

From 1b5753c14c970b388756dad6aaa174faf10840a8 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 21 Aug 2019 19:17:00 -0700
Subject: [PATCH 08/47] Draft EventProcessor Loadbalancing

---
 .../eventprocessor/_ownership_manager.py      | 120 ++++++++++
 .../eventprocessor/event_processor.py         | 207 +++++++++---------
 .../eventprocessor/partition_processor.py     |  15 +-
 .../sqlite3_partition_manager.py              |  95 ++++----
 4 files changed, 287 insertions(+), 150 deletions(-)
 create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
new file mode 100644
index 000000000000..710e71ab8439
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -0,0 +1,120 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# -----------------------------------------------------------------------------------
+
+from typing import List, Iterable, Any, Dict
+import time
+import random
+import math
+from collections import Counter
+from azure.eventhub.aio import EventHubClient
+
+
+class OwnershipManager(object):
+    """Increases or decreases the number of partitions owned by an EventProcessor
+    so the number of owned partitions are balanced among multiple EventProcessors
+
+    An EventProcessor calls claim_ownership() of this class every x seconds,
+    where x is set by keyword argument "polling_interval" in EventProcessor,
+    to claim the ownership of partitions, create tasks for the claimed ownership, and cancel tasks that no longer belong
+    to the claimed ownership.
+
+    """
+    def __init__(self, event_processor, eventhub_client: EventHubClient, ownership_timeout: int):
+        self.all_parition_ids = []
+        self.eventhub_client = eventhub_client
+        self.eventhub_name = eventhub_client.eh_name
+        self.consumer_group_name = event_processor._consumer_group_name
+        self.owner_id = event_processor._id
+        self.partition_manager = event_processor._partition_manager
+        self.ownership_timeout = ownership_timeout
+
+    async def claim_ownership(self):
+        """Claims ownership for this EventProcessor
+        1. Retrieves all partition ids of an event hub from azure event hub service
+        2. Retrieves current ownership list via this EventProcessor's PartitionManager.
+        3. Searches claimable partitions for this EventProcessor. Refer to claim_ownership() for details.
+        4. Claims the ownership for the claimable partitions
+
+        :return: List[Dict[Any]]
+        """
+        if not self.all_parition_ids:
+            await self._retrieve_partition_ids()
+        to_claim = await self._balance_ownership()
+        claimed_list = await self._claim_ownership(to_claim)
+        return claimed_list
+
+    async def _retrieve_partition_ids(self):
+        """List all partition ids of the event hub that the EventProcessor is working on.
+
+        :return: List[str]
+        """
+        self.all_parition_ids = await self.eventhub_client.get_partition_ids()
+
+    async def _balance_ownership(self):
+        ownership_list = await self.partition_manager.list_ownership(self.eventhub_client.eh_name, self.consumer_group_name)
+        ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
+        '''
+        now = time.time()
+        partition_ids_no_ownership = list(filter(lambda x: x not in ownership_dict, self.all_parition_ids))
+        inactive_ownership = filter(lambda x: x["last_modified_time"] + self.ownership_timeout < now, ownership_list)
+        claimable_partition_ids = partition_ids_no_ownership + [x["partition_id"] for x in inactive_ownership]
+        active_ownership = list(filter(lambda x: x["last_modified_time"] + self.ownership_timeout >= now, ownership_list))
+        active_ownership_count_group_by_owner = Counter([x["owner_id"] for x in active_ownership])
+        active_ownership_self = list(filter(lambda x: x["owner_id"] == self.owner_id, active_ownership))
+        '''
+        claimable_partition_ids = []
+        active_ownership_self = []
+        active_ownership_count_group_by_owner = Counter()
+        for partition_id in self.all_parition_ids:
+            ownership = ownership_dict.get(partition_id)
+            if not ownership:  # no ownership found for this partition. So it is claimable
+                claimable_partition_ids.append(partition_id)
+            else:
+                last_modified_time = ownership["last_modified_time"]
+                owner_id = ownership["owner_id"]
+                now = time.time()
+                if now > self.ownership_timeout + last_modified_time:  # ownership timed out. So it is claimable
+                    claimable_partition_ids.append(partition_id)
+                else:  # the ownership is still active
+                    if owner_id == self.owner_id:  # partition is actively owned by this running EventProcessor
+                        active_ownership_self.append(ownership)
+                    active_ownership_count_group_by_owner[owner_id] = active_ownership_count_group_by_owner.get(owner_id, 0) + 1  # all active owners
+
+        # calculate expected count per owner
+        all_partition_count = len(self.all_parition_ids)
+        owners_count = len(active_ownership_count_group_by_owner) + (1 if self.owner_id not in active_ownership_count_group_by_owner else 0)
+        expected_count_per_owner = all_partition_count // owners_count
+        most_count_allowed_per_owner = math.ceil(all_partition_count / owners_count)
+        # end of calculating expected count per owner
+
+        to_claim = active_ownership_self
+        if len(active_ownership_self) > most_count_allowed_per_owner:  # needs to abandon a partition
+            to_claim.pop()  # abandon one partition if owned too many
+            # TODO: Release a ownership immediately so other EventProcessors won't need to wait it to timeout
+        elif len(active_ownership_self) < expected_count_per_owner:  # Either claims an inactive partition, or steals from other owners
+            if claimable_partition_ids:  # claim an inactive partition if there is
+                random_partition_id = random.choice(claimable_partition_ids)
+                random_chosen_to_claim = ownership_dict.get(random_partition_id,
+                                                            {"partition_id": random_partition_id,
+                                                             "eventhub_name": self.eventhub_client.eh_name,
+                                                             "consumer_group_name": self.consumer_group_name,
+                                                             "owner_level": 0})
+                random_chosen_to_claim["owner_id"] = self.owner_id
+                to_claim.append(random_chosen_to_claim)
+            else:  # steal from another owner that has the most count
+                most_frequent_owner_id = active_ownership_count_group_by_owner.most_common(1)[0][0]
+                # randomly choose a partition to steal from the most_frequent_owner
+                to_steal_partition = random.choice(list(filter(lambda x: x["owner_id"] == most_frequent_owner_id,
+                                                   ownership_list)))
+                to_steal_partition["owner_id"] = self.owner_id
+                to_claim.append(to_steal_partition)
+        return to_claim
+
+    async def _claim_ownership(self, ownership_list):
+        if ownership_list:
+            claimed_list = await self.partition_manager.claim_ownership(ownership_list)
+            return claimed_list
+        else:
+            return None
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 85020257df46..c675afd165a4 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -3,15 +3,17 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # -----------------------------------------------------------------------------------
 
-from typing import Callable, List
+from typing import Callable, Dict
 import uuid
 import asyncio
 import logging
+from enum import Enum
 
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
 from .checkpoint_manager import CheckpointManager
 from .partition_manager import PartitionManager
+from ._ownership_manager import OwnershipManager
 from .partition_processor import PartitionProcessor, CloseReason
 from .utils import get_running_loop
 
@@ -53,7 +55,7 @@ async def process_events(self, events):
 
     """
     def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
-                 partition_processor_factory: Callable[[CheckpointManager], PartitionProcessor],
+                 partition_processor_factory,
                  partition_manager: PartitionManager, **kwargs):
         """
         Instantiate an EventProcessor.
@@ -73,16 +75,22 @@ def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
         :type initial_event_position: int or str
 
         """
+
         self._consumer_group_name = consumer_group_name
         self._eventhub_client = eventhub_client
         self._eventhub_name = eventhub_client.eh_name
         self._partition_processor_factory = partition_processor_factory
         self._partition_manager = partition_manager
         self._initial_event_position = kwargs.get("initial_event_position", "-1")
+        # TODO: initial position provider will be a callable
+        #  so users can create initial event position for every partition
         self._max_batch_size = eventhub_client.config.max_batch_size
         self._receive_timeout = eventhub_client.config.receive_timeout
-        self._tasks = []  # type: List[asyncio.Task]
+        self._polling_interval = kwargs.get("polling_interval", 10)
+        self._ownership_timeout = self._polling_interval * 2  # TODO: Team haven't decided if this is a separate argument
+        self._tasks = {}  # type: Dict[str, asyncio.Task]
         self._id = str(uuid.uuid4())
+        self._running = False
 
     def __repr__(self):
         return 'EventProcessor: id {}'.format(self._id)
@@ -98,9 +106,22 @@ async def start(self):
 
         """
         log.info("EventProcessor %r is being started", self._id)
-        partition_ids = await self._eventhub_client.get_partition_ids()
-        claimed_list = await self._claim_partitions(partition_ids)
-        await self._start_claimed_partitions(claimed_list)
+        ownership_manager = OwnershipManager(self, self._eventhub_client, self._ownership_timeout)
+        if not self._running:
+            self._running = True
+            while self._running:
+                claimed_ownership_list = await ownership_manager.claim_ownership()
+                claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
+                to_cancel_list = self._tasks.keys() - claimed_partition_ids
+                if to_cancel_list:
+                    self._cancel_tasks_for_partitions(to_cancel_list)
+                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
+
+                if claimed_partition_ids:
+                    self._create_tasks_for_claimed_ownership(claimed_ownership_list)
+                else:
+                    log.warning("EventProcessor %r hasn't claimed an ownership. It keeps claiming.", self._id)
+                await asyncio.sleep(self._polling_interval)
 
     async def stop(self):
         """Stop all the partition consumer
@@ -110,109 +131,85 @@ async def stop(self):
         :return: None
 
         """
+        self._running = False
         for i in range(len(self._tasks)):
-            task = self._tasks.pop()
+            task = self._tasks.popitem()[1]
             task.cancel()
         log.info("EventProcessor %r has been cancelled", self._id)
         await asyncio.sleep(2)  # give some time to finish after cancelled
 
-    async def _claim_partitions(self, partition_ids):
-        partitions_ownership = await self._partition_manager.list_ownership(self._eventhub_name, self._consumer_group_name)
-        partitions_ownership_dict = dict()
-        for ownership in partitions_ownership:
-            partitions_ownership_dict[ownership["partition_id"]] = ownership
-
-        to_claim_list = []
-        for pid in partition_ids:
-            p_ownership = partitions_ownership_dict.get(pid)
-            if p_ownership:
-                to_claim_list.append(p_ownership)
-            else:
-                new_ownership = {"eventhub_name": self._eventhub_name, "consumer_group_name": self._consumer_group_name,
-                                 "owner_id": self._id, "partition_id": pid, "owner_level": OWNER_LEVEL}
-                to_claim_list.append(new_ownership)
-        claimed_list = await self._partition_manager.claim_ownership(to_claim_list)
-        return claimed_list
-
-    async def _start_claimed_partitions(self, claimed_partitions):
-        for partition in claimed_partitions:
-            partition_id = partition["partition_id"]
-            offset = partition.get("offset", self._initial_event_position)
-            consumer = self._eventhub_client.create_consumer(self._consumer_group_name, partition_id,
-                                                            EventPosition(str(offset)))
-            partition_processor = self._partition_processor_factory(
-                checkpoint_manager=CheckpointManager(partition_id, self._eventhub_name, self._consumer_group_name,
-                                                     self._id, self._partition_manager)
-            )
-            loop = get_running_loop()
-            task = loop.create_task(
-                _receive(consumer, partition_processor, self._receive_timeout))
-            self._tasks.append(task)
+    def _cancel_tasks_for_partitions(self, to_cancel_partitions):
+        for partition_id in to_cancel_partitions:
+            if partition_id in self._tasks:
+                task = self._tasks.pop(partition_id)
+                task.cancel()
+
+    def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list):
+        for ownership in to_claim_ownership_list:
+            partition_id = ownership["partition_id"]
+            if partition_id not in self._tasks:
+                self._tasks[partition_id] = asyncio.create_task(self._receive(ownership))
+
+    async def _receive(self, ownership):
+        log.info("start ownership, %r", ownership)
+        partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
+                                                                   ownership["partition_id"],
+                                                                   EventPosition(ownership.get("offset", self._initial_event_position))
+                                                                   )
+        checkpoint_manager = CheckpointManager(ownership["partition_id"],
+                                               ownership["eventhub_name"],
+                                               ownership["consumer_group_name"],
+                                               ownership["owner_id"],
+                                               self._partition_manager)
+        partition_processor = self._partition_processor_factory()
+
+        async def initialize():
+            if hasattr(partition_processor, "initialize"):
+                await partition_processor.initialize(checkpoint_manager)
+
+        async def process_error(err):
+            if hasattr(partition_processor, "process_error"):
+                await partition_processor.process_error(err, checkpoint_manager)
+
+        async def close(close_reason):
+            if hasattr(partition_processor, "close"):
+                await partition_processor.close(close_reason, checkpoint_manager)
+
         try:
-            await asyncio.gather(*self._tasks)
+            while True:
+                try:
+                    await initialize()
+                    events = await partition_consumer.receive(timeout=self._receive_timeout)
+                    await partition_processor.process_events(events, checkpoint_manager)
+                except asyncio.CancelledError as cancelled_error:
+                    log.info(
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
+                        "is cancelled",
+                        ownership["owner_id"],
+                        ownership["eventhub_name"],
+                        ownership["partition_id"],
+                        ownership["consumer_group_name"]
+                    )
+                    await process_error(cancelled_error)
+                    await close(CloseReason.SHUTDOWN)
+                    break
+                except EventHubError as eh_err:
+                    reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
+                    log.warning(
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
+                        "has met an exception receiving events. It's being closed. The exception is %r.",
+                        ownership["owner_id"],
+                        ownership["eventhub_name"],
+                        ownership["partition_id"],
+                        ownership["consumer_group_name"],
+                        eh_err
+                    )
+                    await process_error(eh_err)
+                    await close(reason)
+                    break
+                except Exception as exp:
+                    log.warning(exp)
+                    # TODO: will review whether to break and close partition processor after user's code has an exception
+            # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
         finally:
-            log.info("EventProcessor %r has stopped", self._id)
-
-
-async def _receive(partition_consumer, partition_processor, receive_timeout):
-    try:
-        while True:
-            try:
-                events = await partition_consumer.receive(timeout=receive_timeout)
-            except asyncio.CancelledError as cancelled_error:
-                log.info(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "is cancelled",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name
-                )
-                await partition_processor.process_error(cancelled_error)
-                await partition_processor.close(reason=CloseReason.SHUTDOWN)
-                break
-            except EventHubError as eh_err:
-                reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
-                log.warning(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "has met an exception receiving events. It's being closed. The exception is %r.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name,
-                    eh_err
-                )
-                await partition_processor.process_error(eh_err)
-                await partition_processor.close(reason=reason)
-                break
-            try:
-                await partition_processor.process_events(events)
-            except asyncio.CancelledError as cancelled_error:
-                log.info(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "is cancelled.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name
-                )
-                await partition_processor.process_error(cancelled_error)
-                await partition_processor.close(reason=CloseReason.SHUTDOWN)
-                break
-            except Exception as exp:  # user code has caused an error
-                log.warning(
-                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                    "has met an exception from user code process_events. It's being closed. The exception is %r.",
-                    partition_processor._checkpoint_manager.owner_id,
-                    partition_processor._checkpoint_manager.eventhub_name,
-                    partition_processor._checkpoint_manager.partition_id,
-                    partition_processor._checkpoint_manager.consumer_group_name,
-                    exp
-                )
-                await partition_processor.process_error(exp)
-                await partition_processor.close(reason=CloseReason.EVENTHUB_EXCEPTION)
-                break
-                # TODO: will review whether to break and close partition processor after user's code has an exception
-        # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
-    finally:
-        await partition_consumer.close()
+            await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 10aafc79c492..94729aae1269 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -4,7 +4,7 @@
 # -----------------------------------------------------------------------------------
 
 from typing import List
-from abc import ABC, abstractmethod
+from typing_extensions import Protocol
 from enum import Enum
 from .checkpoint_manager import CheckpointManager
 
@@ -17,16 +17,16 @@ class CloseReason(Enum):
     EVENTHUB_EXCEPTION = 2  # Exception happens during receiving events
 
 
-class PartitionProcessor(ABC):
+class PartitionProcessor(Protocol):
     """
     PartitionProcessor processes events received from the Azure Event Hubs service. A single instance of a class
     implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns.
 
     """
-    def __init__(self, checkpoint_manager: CheckpointManager):
-        self._checkpoint_manager = checkpoint_manager
+    async def initialize(self, checkpoint_manager: CheckpointManager):
+        pass
 
-    async def close(self, reason):
+    async def close(self, reason, checkpoint_manager: CheckpointManager):
         """Called when EventProcessor stops processing this PartitionProcessor.
 
         There are different reasons to trigger the PartitionProcessor to close.
@@ -38,8 +38,7 @@ async def close(self, reason):
         """
         pass
 
-    @abstractmethod
-    async def process_events(self, events: List[EventData]):
+    async def process_events(self, events: List[EventData], checkpoint_manager: CheckpointManager):
         """Called when a batch of events have been received.
 
         :param events: Received events.
@@ -48,7 +47,7 @@ async def process_events(self, events: List[EventData]):
         """
         pass
 
-    async def process_error(self, error):
+    async def process_error(self, error, checkpoint_manager: CheckpointManager):
         """Called when an error happens
 
         :param error: The error that happens.
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index eb08e970fa89..5e2c3dd45748 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -6,8 +6,11 @@
 import time
 import uuid
 import sqlite3
+import logging
 from .partition_manager import PartitionManager
 
+logger = logging.getLogger(__name__)
+
 
 def _check_table_name(table_name: str):
     for c in table_name:
@@ -22,6 +25,15 @@ class Sqlite3PartitionManager(PartitionManager):
 
 
     """
+    primary_keys_dict = {"eventhub_name": "text", "consumer_group_name": "text", "partition_id": "text"}
+    other_fields_dict = {"owner_id": "text", "owner_level": "integer", "sequence_number": "integer", "offset": "text",
+                         "last_modified_time": "real", "etag": "text"}
+    checkpoint_fields = ["sequence_number", "offset"]
+    fields_dict = {**primary_keys_dict, **other_fields_dict}
+    primary_keys = list(primary_keys_dict.keys())
+    other_fields = list(other_fields_dict.keys())
+    fields = primary_keys + other_fields
+
     def __init__(self, db_filename: str = ":memory:", ownership_table: str = "ownership"):
         """
 
@@ -34,17 +46,15 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
         conn = sqlite3.connect(db_filename)
         c = conn.cursor()
         try:
-            c.execute("create table " + ownership_table +
-                      "(eventhub_name text,"
-                      "consumer_group_name text,"
-                      "owner_id text,"
-                      "partition_id text,"
-                      "owner_level integer,"
-                      "sequence_number integer,"
-                      "offset text,"
-                      "last_modified_time integer,"
-                      "etag text)")
+            sql = "create table if not exists " + _check_table_name(ownership_table)\
+                  + "("\
+                  + ",".join([x[0]+" "+x[1] for x in self.fields_dict.items()])\
+                  + ", constraint pk_ownership PRIMARY KEY ("\
+                  + ",".join(self.primary_keys)\
+                  + "))"
+            c.execute(sql)
         except sqlite3.OperationalError:
+            raise
             pass
         finally:
             c.close()
@@ -53,44 +63,55 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
     async def list_ownership(self, eventhub_name, consumer_group_name):
         cursor = self.conn.cursor()
         try:
-            fields = ["eventhub_name", "consumer_group_name", "owner_id", "partition_id", "owner_level",
-                      "sequence_number",
-                      "offset", "last_modified_time", "etag"]
-            cursor.execute("select " + ",".join(fields) +
+            cursor.execute("select " + ",".join(self.fields) +
                                 " from "+_check_table_name(self.ownership_table)+" where eventhub_name=? "
                                 "and consumer_group_name=?",
                                 (eventhub_name, consumer_group_name))
-            result_list = []
-
-            for row in cursor.fetchall():
-                d = dict(zip(fields, row))
-                result_list.append(d)
-            return result_list
+            return [dict(zip(self.fields, row)) for row in cursor.fetchall()]
         finally:
             cursor.close()
 
     async def claim_ownership(self, partitions):
+        result = []
         cursor = self.conn.cursor()
         try:
             for p in partitions:
-                cursor.execute("select * from " + _check_table_name(self.ownership_table) +
-                                    " where eventhub_name=? "
-                                    "and consumer_group_name=? "
-                                    "and partition_id =?",
-                                    (p["eventhub_name"], p["consumer_group_name"],
-                                     p["partition_id"]))
-                if not cursor.fetchall():
-                    cursor.execute("insert into " + _check_table_name(self.ownership_table) +
-                                   " (eventhub_name,consumer_group_name,partition_id,owner_id,owner_level,last_modified_time,etag) "
-                                   "values (?,?,?,?,?,?,?)",
-                                   (p["eventhub_name"], p["consumer_group_name"], p["partition_id"], p["owner_id"], p["owner_level"],
-                                    time.time(), str(uuid.uuid4())
-                                    ))
+                cursor.execute("select etag from " + _check_table_name(self.ownership_table) +
+                                    " where "+ " and ".join([field+"=?" for field in self.primary_keys]),
+                                    tuple(p.get(field) for field in self.primary_keys))
+                cursor_fetch = cursor.fetchall()
+                if not cursor_fetch:
+                    p["last_modified_time"] = time.time()
+                    p["etag"] = str(uuid.uuid4())
+                    try:
+                        fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.fields))
+                        sql = "insert into " + _check_table_name(self.ownership_table) + " (" \
+                              + ",".join(fields_without_checkpoint) \
+                              + ") values (?,?,?,?,?,?,?)"
+                        cursor.execute(sql, tuple(p.get(field) for field in fields_without_checkpoint))
+                    except sqlite3.OperationalError as op_err:
+                        logger.info("EventProcessor %r failed to claim partition %r "
+                                    "because it was claimed by another EventProcessor at the same time. "
+                                    "The Sqlite3 exception is %r", p["owner_id"], p["partition_id"], op_err)
+                        break
+                    else:
+                        result.append(p)
                 else:
-                    cursor.execute("update " + _check_table_name(self.ownership_table) + " set owner_id=?, owner_level=?, last_modified_time=?, etag=? "
-                                   "where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                                   (p["owner_id"], p["owner_level"], time.time(), str(uuid.uuid4()),
-                                    p["eventhub_name"], p["consumer_group_name"], p["partition_id"]))
+                    if p.get("etag") == cursor_fetch[0][0]:
+                        p["last_modified_time"] = time.time()
+                        p["etag"] = str(uuid.uuid4())
+                        other_fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.other_fields))
+                        sql = "update " + _check_table_name(self.ownership_table) + " set "\
+                                       + ','.join([field+"=?" for field in other_fields_without_checkpoint])\
+                                       + " where "\
+                                       + " and ".join([field+"=?" for field in self.primary_keys])
+
+                        cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint) + tuple(p.get(field) for field in self.primary_keys))
+                        result.append(p)
+                    else:
+                        logger.info("EventProcessor %r failed to claim partition %r "
+                                    "because it was claimed by another EventProcessor at the same time", p["owner_id"],
+                                    p["partition_id"])
             self.conn.commit()
             return partitions
         finally:

From b4b77f9d6bb19fb2f4f7e6fa957c25fe572a4fec Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 01:01:35 -0700
Subject: [PATCH 09/47] EventProcessor Load balancing

---
 .../eventprocessor/_ownership_manager.py      | 45 ++++++++-----
 .../eventprocessor/event_processor.py         | 66 +++++++++++--------
 .../eventprocessor/partition_processor.py     |  4 +-
 .../sqlite3_partition_manager.py              | 13 +++-
 .../eventprocessor/event_processor_example.py | 39 ++++-------
 5 files changed, 94 insertions(+), 73 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
index 710e71ab8439..40e3419a7c5a 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # -----------------------------------------------------------------------------------
 
-from typing import List, Iterable, Any, Dict
 import time
 import random
 import math
@@ -34,8 +33,8 @@ async def claim_ownership(self):
         """Claims ownership for this EventProcessor
         1. Retrieves all partition ids of an event hub from azure event hub service
         2. Retrieves current ownership list via this EventProcessor's PartitionManager.
-        3. Searches claimable partitions for this EventProcessor. Refer to claim_ownership() for details.
-        4. Claims the ownership for the claimable partitions
+        3. Balances number of ownership. Refer to _balance_ownership() for details.
+        4. Claims the ownership for the balanced number of partitions.
 
         :return: List[Dict[Any]]
         """
@@ -53,19 +52,35 @@ async def _retrieve_partition_ids(self):
         self.all_parition_ids = await self.eventhub_client.get_partition_ids()
 
     async def _balance_ownership(self):
+        """Balances and claims ownership of partitions for this EventProcessor.
+        The balancing algorithm is:
+        1. Find partitions with inactive ownership and partitions that haven never been claimed before
+        2. Find the number of active owners, including this EventProcessor, for all partitions.
+        3. Calculate the average count of partitions that an owner should own.
+        (number of partitions // number of active owners)
+        4. Calculate the largest allowed count of partitions that an owner can own.
+        math.ceil(number of partitions / number of active owners). This should be equal or 1 greater than the average count
+        5. Adjust the number of partitions owned by this EventProcessor (owner)
+            a. if this EventProcessor owns more than largest allowed count, abandon one partition
+            b. if this EventProcessor owns less than average count, add one from the inactive or unclaimed partitions,
+                or steal one from another owner that has the largest number of ownership among all owners (EventProcessors)
+            c. Otherwise, no change to the ownership
+
+        The balancing algorithm adjust one partition at a time to gradually build the balanced ownership.
+        Ownership must be renewed to keep it active. So the returned result includes both existing ownership and
+        the newly adjusted ownership.
+        This method balances but doesn't claim ownership. The caller of this method tries to claim the result ownership
+        list. But it may not successfully claim all of them because of concurrency. Other EventProcessors may happen to
+        claim a partition at that time. Since balancing and claiming are run in infinite repeatedly,
+        it achieves balancing among all EventProcessors after some time of running.
+
+        :return: List[Dict[str, Any]], A list of ownership.
+        """
         ownership_list = await self.partition_manager.list_ownership(self.eventhub_client.eh_name, self.consumer_group_name)
         ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
-        '''
-        now = time.time()
-        partition_ids_no_ownership = list(filter(lambda x: x not in ownership_dict, self.all_parition_ids))
-        inactive_ownership = filter(lambda x: x["last_modified_time"] + self.ownership_timeout < now, ownership_list)
-        claimable_partition_ids = partition_ids_no_ownership + [x["partition_id"] for x in inactive_ownership]
-        active_ownership = list(filter(lambda x: x["last_modified_time"] + self.ownership_timeout >= now, ownership_list))
-        active_ownership_count_group_by_owner = Counter([x["owner_id"] for x in active_ownership])
-        active_ownership_self = list(filter(lambda x: x["owner_id"] == self.owner_id, active_ownership))
-        '''
-        claimable_partition_ids = []
-        active_ownership_self = []
+
+        claimable_partition_ids = []  # partitions with inactive ownership and partitions that have never been claimed yet
+        active_ownership_self = []  # active ownership of this EventProcessor
         active_ownership_count_group_by_owner = Counter()
         for partition_id in self.all_parition_ids:
             ownership = ownership_dict.get(partition_id)
@@ -100,7 +115,7 @@ async def _balance_ownership(self):
                                                             {"partition_id": random_partition_id,
                                                              "eventhub_name": self.eventhub_client.eh_name,
                                                              "consumer_group_name": self.consumer_group_name,
-                                                             "owner_level": 0})
+                                                             "owner_level": 0})  # TODO: consider removing owner_level
                 random_chosen_to_claim["owner_id"] = self.owner_id
                 to_claim.append(random_chosen_to_claim)
             else:  # steal from another owner that has the most count
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index c675afd165a4..7e342a369d0d 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -7,15 +7,13 @@
 import uuid
 import asyncio
 import logging
-from enum import Enum
 
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
 from .checkpoint_manager import CheckpointManager
 from .partition_manager import PartitionManager
 from ._ownership_manager import OwnershipManager
-from .partition_processor import PartitionProcessor, CloseReason
-from .utils import get_running_loop
+from .partition_processor import CloseReason, PartitionProcessor
 
 log = logging.getLogger(__name__)
 
@@ -29,20 +27,21 @@ class EventProcessor(object):
 
     It provides the user a convenient way to receive events from multiple partitions and save checkpoints.
     If multiple EventProcessors are running for an event hub, they will automatically balance load.
-    This load balancing won't be available until preview 3.
 
     Example:
         .. code-block:: python
 
-            class MyPartitionProcessor(PartitionProcessor):
-                async def process_events(self, events):
-                    if events:
-                        # do something sync or async to process the events
-                        await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
-
             import asyncio
             from azure.eventhub.aio import EventHubClient
             from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager
+
+            class MyPartitionProcessor(object):
+                async def process_events(self, events, checkpoint_manager):
+                    if events:
+                        # do something sync or async to process the events
+                        await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+
+
             client = EventHubClient.from_connection_string("<your connection string>", receive_timeout=5, retry_total=3)
             partition_manager = Sqlite3PartitionManager()
             try:
@@ -55,7 +54,7 @@ async def process_events(self, events):
 
     """
     def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
-                 partition_processor_factory,
+                 partition_processor_factory: Callable[..., PartitionProcessor],
                  partition_manager: PartitionManager, **kwargs):
         """
         Instantiate an EventProcessor.
@@ -73,6 +72,8 @@ def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
         :type partition_manager: Class implementing the ~azure.eventhub.eventprocessor.PartitionManager.
         :param initial_event_position: The offset to start a partition consumer if the partition has no checkpoint yet.
         :type initial_event_position: int or str
+        :param polling_interval: The interval between any two pollings of balancing and claiming
+        :type float
 
         """
 
@@ -98,9 +99,12 @@ def __repr__(self):
     async def start(self):
         """Start the EventProcessor.
 
-        1. retrieve the partition ids from eventhubs.
-        2. claim partition ownership of these partitions.
-        3. repeatedly call EvenHubConsumer.receive() to retrieve events and call user defined PartitionProcessor.process_events().
+        1. Calls the OwnershipManager to keep claiming and balancing ownership of partitions in an
+        infinitely loop until self.stop() is called.
+        2. Cancels tasks for partitions that are no longer owned by this EventProcessor
+        3. Creates tasks for partitions that are newly claimed by this EventProcessor
+        4. Keeps tasks running for partitions that haven't changed ownership
+        5. Each task repeatedly calls EvenHubConsumer.receive() to retrieve events and call user defined partition processor
 
         :return: None
 
@@ -111,22 +115,23 @@ async def start(self):
             self._running = True
             while self._running:
                 claimed_ownership_list = await ownership_manager.claim_ownership()
-                claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
-                to_cancel_list = self._tasks.keys() - claimed_partition_ids
-                if to_cancel_list:
-                    self._cancel_tasks_for_partitions(to_cancel_list)
-                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
-
-                if claimed_partition_ids:
+                if claimed_ownership_list:
+                    claimed_partition_ids = [x["partition_id"] for x in claimed_ownership_list]
+                    to_cancel_list = self._tasks.keys() - claimed_partition_ids
                     self._create_tasks_for_claimed_ownership(claimed_ownership_list)
                 else:
                     log.warning("EventProcessor %r hasn't claimed an ownership. It keeps claiming.", self._id)
+                    to_cancel_list = self._tasks.keys()
+                if to_cancel_list:
+                    self._cancel_tasks_for_partitions(to_cancel_list)
+                    log.info("EventProcesor %r has cancelled partitions %r", self._id, to_cancel_list)
                 await asyncio.sleep(self._polling_interval)
 
     async def stop(self):
-        """Stop all the partition consumer
+        """Stop claiming ownership and all the partition consumers owned by this EventProcessor
 
-        This method cancels tasks that are running EventHubConsumer.receive() for the partitions owned by this EventProcessor.
+        This method stops claiming ownership of owned partitions and cancels tasks that are running
+        EventHubConsumer.receive() for the partitions owned by this EventProcessor.
 
         :return: None
 
@@ -152,6 +157,13 @@ def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list):
 
     async def _receive(self, ownership):
         log.info("start ownership, %r", ownership)
+        partition_processor = self._partition_processor_factory()
+        if not hasattr(partition_processor, "process_events"):
+            log.error(
+                "Fatal error: a partition processor should at least have method process_events(events, checkpoint_manager). EventProcessor will stop.")
+            await self.stop()
+            raise TypeError("Partition processor must has method process_events(events, checkpoint_manager")
+
         partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
                                                                    ownership["partition_id"],
                                                                    EventPosition(ownership.get("offset", self._initial_event_position))
@@ -161,8 +173,6 @@ async def _receive(self, ownership):
                                                ownership["consumer_group_name"],
                                                ownership["owner_id"],
                                                self._partition_manager)
-        partition_processor = self._partition_processor_factory()
-
         async def initialize():
             if hasattr(partition_processor, "initialize"):
                 await partition_processor.initialize(checkpoint_manager)
@@ -192,6 +202,7 @@ async def close(close_reason):
                     )
                     await process_error(cancelled_error)
                     await close(CloseReason.SHUTDOWN)
+                    # TODO: release the ownership immediately via partition manager
                     break
                 except EventHubError as eh_err:
                     reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
@@ -205,11 +216,10 @@ async def close(close_reason):
                         eh_err
                     )
                     await process_error(eh_err)
-                    await close(reason)
+                    await close(reason)  # An EventProcessor will pick up this partition again after the ownership is released
+                    # TODO: release the ownership immediately via partition manager
                     break
                 except Exception as exp:
                     log.warning(exp)
-                    # TODO: will review whether to break and close partition processor after user's code has an exception
-            # TODO: try to inform other EventProcessors to take the partition when this partition is closed in preview 3?
         finally:
             await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 94729aae1269..60a95b99e628 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -23,7 +23,7 @@ class PartitionProcessor(Protocol):
     implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns.
 
     """
-    async def initialize(self, checkpoint_manager: CheckpointManager):
+    async def initialize(self):
         pass
 
     async def close(self, reason, checkpoint_manager: CheckpointManager):
@@ -45,7 +45,7 @@ async def process_events(self, events: List[EventData], checkpoint_manager: Chec
         :type events: list[~azure.eventhub.common.EventData]
 
         """
-        pass
+        raise NotImplementedError
 
     async def process_error(self, error, checkpoint_manager: CheckpointManager):
         """Called when an error happens
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index 5e2c3dd45748..bf4c46c95eb9 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -121,9 +121,16 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             offset, sequence_number):
         cursor = self.conn.cursor()
         try:
-            cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                           (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
-            self.conn.commit()
+            cursor.execute("select owner_id from " + _check_table_name(self.ownership_table) + " where eventhub_name=? and consumer_group_name=? and partition_id=?",
+                           (eventhub_name, consumer_group_name, partition_id))
+            cursor_fetch = cursor.fetchall()
+            if cursor_fetch and owner_id == cursor_fetch[0][0]:
+                cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?",
+                               (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
+                self.conn.commit()
+            else:
+                logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership", partition_id)
+
         finally:
             cursor.close()
 
diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
index 8c4c9ced7d29..a2165eb81110 100644
--- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
+++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
@@ -3,7 +3,6 @@
 import os
 from azure.eventhub.aio import EventHubClient
 from azure.eventhub.eventprocessor import EventProcessor
-from azure.eventhub.eventprocessor import PartitionProcessor
 from azure.eventhub.eventprocessor import Sqlite3PartitionManager
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
@@ -18,33 +17,23 @@ async def do_operation(event):
     print(event)
 
 
-class MyPartitionProcessor(PartitionProcessor):
-    def __init__(self, checkpoint_manager):
-        super(MyPartitionProcessor, self).__init__(checkpoint_manager)
-
-    async def process_events(self, events):
+class MyPartitionProcessor(object):
+    async def process_events(self, events, checkpoint_manager):
         if events:
             await asyncio.gather(*[do_operation(event) for event in events])
-            await self._checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
-
-
-def partition_processor_factory(checkpoint_manager):
-    return MyPartitionProcessor(checkpoint_manager)
-
-
-async def run_awhile(duration):
-    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT,
-                                                   retry_total=RETRY_TOTAL)
-    partition_manager = Sqlite3PartitionManager()
-    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager)
-    try:
-        asyncio.ensure_future(event_processor.start())
-        await asyncio.sleep(duration)
-        await event_processor.stop()
-    finally:
-        await partition_manager.close()
+            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+        else:
+            print("empty events received", "partition:", checkpoint_manager.partition_id)
 
 
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(run_awhile(60))
+    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
+    partition_manager = Sqlite3PartitionManager(db_filename="eventprocessor_test_db")
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
+    try:
+        loop.run_until_complete(event_processor.start())
+    except KeyboardInterrupt:
+        loop.run_until_complete(event_processor.stop())
+    finally:
+        loop.stop()

From 1787fdd6ff1c9339e67a6bf76c5aa4513506f2ff Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:10:35 -0700
Subject: [PATCH 10/47] small changes from bryan's review

---
 .../azure/eventhub/eventprocessor/event_processor.py        | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 7e342a369d0d..2db6feb4bbf1 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -137,8 +137,8 @@ async def stop(self):
 
         """
         self._running = False
-        for i in range(len(self._tasks)):
-            task = self._tasks.popitem()[1]
+        for _ in range(len(self._tasks)):
+            _, task = self._tasks.popitem()
             task.cancel()
         log.info("EventProcessor %r has been cancelled", self._id)
         await asyncio.sleep(2)  # give some time to finish after cancelled
@@ -205,7 +205,7 @@ async def close(close_reason):
                     # TODO: release the ownership immediately via partition manager
                     break
                 except EventHubError as eh_err:
-                    reason = CloseReason.LEASE_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
+                    reason = CloseReason.OWNERSHIP_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
                     log.warning(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
                         "has met an exception receiving events. It's being closed. The exception is %r.",

From c2d0155c58d3e74f28931538c44d4e9f7af4d25d Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:21:05 -0700
Subject: [PATCH 11/47] remove checkpoint manager from initialize

---
 .../azure/eventhub/eventprocessor/event_processor.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 2db6feb4bbf1..e808d603c457 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -175,7 +175,7 @@ async def _receive(self, ownership):
                                                self._partition_manager)
         async def initialize():
             if hasattr(partition_processor, "initialize"):
-                await partition_processor.initialize(checkpoint_manager)
+                await partition_processor.initialize()
 
         async def process_error(err):
             if hasattr(partition_processor, "process_error"):

From 10743856d30facb239bc4b14e54e6a5ec38a934b Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 22 Aug 2019 18:41:18 -0700
Subject: [PATCH 12/47] small changes

---
 .../eventprocessor/event_processor.py         | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index e808d603c457..23178deba76a 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -164,15 +164,19 @@ async def _receive(self, ownership):
             await self.stop()
             raise TypeError("Partition processor must has method process_events(events, checkpoint_manager")
 
-        partition_consumer = self._eventhub_client.create_consumer(ownership["consumer_group_name"],
-                                                                   ownership["partition_id"],
-                                                                   EventPosition(ownership.get("offset", self._initial_event_position))
-                                                                   )
-        checkpoint_manager = CheckpointManager(ownership["partition_id"],
-                                               ownership["eventhub_name"],
-                                               ownership["consumer_group_name"],
-                                               ownership["owner_id"],
-                                               self._partition_manager)
+        partition_consumer = self._eventhub_client.create_consumer(
+            ownership["consumer_group_name"],
+            ownership["partition_id"],
+            EventPosition(ownership.get("offset", self._initial_event_position))
+        )
+        checkpoint_manager = CheckpointManager(
+            ownership["partition_id"],
+            ownership["eventhub_name"],
+            ownership["consumer_group_name"],
+            ownership["owner_id"],
+            self._partition_manager
+        )
+
         async def initialize():
             if hasattr(partition_processor, "initialize"):
                 await partition_processor.initialize()

From 386baf011acce33839479ad8ecbf1278e641a993 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 29 Aug 2019 12:14:28 -0700
Subject: [PATCH 13/47] Fix code review feedback

---
 .../eventprocessor/_ownership_manager.py      | 76 +++++++++----------
 .../eventprocessor/checkpoint_manager.py      |  9 ++-
 .../eventprocessor/event_processor.py         | 17 +++--
 .../azure/eventhub/eventprocessor/utils.py    |  2 +-
 4 files changed, 54 insertions(+), 50 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
index 40e3419a7c5a..5a49a1d093e9 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -6,8 +6,10 @@
 import time
 import random
 import math
+import collections
 from collections import Counter
 from azure.eventhub.aio import EventHubClient
+from .partition_manager import PartitionManager
 
 
 class OwnershipManager(object):
@@ -20,13 +22,16 @@ class OwnershipManager(object):
     to the claimed ownership.
 
     """
-    def __init__(self, event_processor, eventhub_client: EventHubClient, ownership_timeout: int):
-        self.all_parition_ids = []
+    def __init__(
+            self, eventhub_client: EventHubClient, consumer_group_name: str, owner_id: str,
+            partition_manager: PartitionManager, ownership_timeout: int
+    ):
+        self.cached_parition_ids = []
         self.eventhub_client = eventhub_client
         self.eventhub_name = eventhub_client.eh_name
-        self.consumer_group_name = event_processor._consumer_group_name
-        self.owner_id = event_processor._id
-        self.partition_manager = event_processor._partition_manager
+        self.consumer_group_name = consumer_group_name
+        self.owner_id = owner_id
+        self.partition_manager = partition_manager
         self.ownership_timeout = ownership_timeout
 
     async def claim_ownership(self):
@@ -38,10 +43,10 @@ async def claim_ownership(self):
 
         :return: List[Dict[Any]]
         """
-        if not self.all_parition_ids:
+        if not self.cached_parition_ids:
             await self._retrieve_partition_ids()
         to_claim = await self._balance_ownership()
-        claimed_list = await self._claim_ownership(to_claim)
+        claimed_list = await self.partition_manager.claim_ownership(to_claim) if to_claim else None
         return claimed_list
 
     async def _retrieve_partition_ids(self):
@@ -49,7 +54,7 @@ async def _retrieve_partition_ids(self):
 
         :return: List[str]
         """
-        self.all_parition_ids = await self.eventhub_client.get_partition_ids()
+        self.cached_parition_ids = await self.eventhub_client.get_partition_ids()
 
     async def _balance_ownership(self):
         """Balances and claims ownership of partitions for this EventProcessor.
@@ -59,11 +64,12 @@ async def _balance_ownership(self):
         3. Calculate the average count of partitions that an owner should own.
         (number of partitions // number of active owners)
         4. Calculate the largest allowed count of partitions that an owner can own.
-        math.ceil(number of partitions / number of active owners). This should be equal or 1 greater than the average count
+        math.ceil(number of partitions / number of active owners).
+        This should be equal or 1 greater than the average count
         5. Adjust the number of partitions owned by this EventProcessor (owner)
             a. if this EventProcessor owns more than largest allowed count, abandon one partition
             b. if this EventProcessor owns less than average count, add one from the inactive or unclaimed partitions,
-                or steal one from another owner that has the largest number of ownership among all owners (EventProcessors)
+            or steal one from another owner that has the largest number of ownership among all owners (EventProcessors)
             c. Otherwise, no change to the ownership
 
         The balancing algorithm adjust one partition at a time to gradually build the balanced ownership.
@@ -76,30 +82,26 @@ async def _balance_ownership(self):
 
         :return: List[Dict[str, Any]], A list of ownership.
         """
-        ownership_list = await self.partition_manager.list_ownership(self.eventhub_client.eh_name, self.consumer_group_name)
+        ownership_list = await self.partition_manager.list_ownership(
+            self.eventhub_client.eh_name, self.consumer_group_name
+        )
+        now = time.time()
         ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
-
-        claimable_partition_ids = []  # partitions with inactive ownership and partitions that have never been claimed yet
-        active_ownership_self = []  # active ownership of this EventProcessor
-        active_ownership_count_group_by_owner = Counter()
-        for partition_id in self.all_parition_ids:
-            ownership = ownership_dict.get(partition_id)
-            if not ownership:  # no ownership found for this partition. So it is claimable
-                claimable_partition_ids.append(partition_id)
-            else:
-                last_modified_time = ownership["last_modified_time"]
-                owner_id = ownership["owner_id"]
-                now = time.time()
-                if now > self.ownership_timeout + last_modified_time:  # ownership timed out. So it is claimable
-                    claimable_partition_ids.append(partition_id)
-                else:  # the ownership is still active
-                    if owner_id == self.owner_id:  # partition is actively owned by this running EventProcessor
-                        active_ownership_self.append(ownership)
-                    active_ownership_count_group_by_owner[owner_id] = active_ownership_count_group_by_owner.get(owner_id, 0) + 1  # all active owners
+        not_owned_partition_ids = [pid for pid in self.cached_parition_ids if pid not in ownership_dict]
+        timed_out_partition_ids = [ownership["partition_id"] for ownership in ownership_list
+                                   if ownership["last_modified_time"] + self.ownership_timeout < now]
+        claimable_partition_ids = not_owned_partition_ids + timed_out_partition_ids
+        active_ownership = [ownership for ownership in ownership_list
+                            if ownership["last_modified_time"] + self.ownership_timeout >= now]
+        active_ownership_by_owner = collections.defaultdict(list)
+        for ownership in active_ownership:
+            active_ownership_by_owner[ownership["owner_id"]].append(ownership)
+        active_ownership_self = active_ownership_by_owner[self.owner_id]
 
         # calculate expected count per owner
-        all_partition_count = len(self.all_parition_ids)
-        owners_count = len(active_ownership_count_group_by_owner) + (1 if self.owner_id not in active_ownership_count_group_by_owner else 0)
+        all_partition_count = len(self.cached_parition_ids)
+        owners_count = len(active_ownership_by_owner) + \
+                       (0 if self.owner_id in active_ownership_by_owner else 1)
         expected_count_per_owner = all_partition_count // owners_count
         most_count_allowed_per_owner = math.ceil(all_partition_count / owners_count)
         # end of calculating expected count per owner
@@ -119,17 +121,11 @@ async def _balance_ownership(self):
                 random_chosen_to_claim["owner_id"] = self.owner_id
                 to_claim.append(random_chosen_to_claim)
             else:  # steal from another owner that has the most count
+                active_ownership_count_group_by_owner = Counter(
+                    (x, len(y)) for x, y in active_ownership_by_owner.items())
                 most_frequent_owner_id = active_ownership_count_group_by_owner.most_common(1)[0][0]
                 # randomly choose a partition to steal from the most_frequent_owner
-                to_steal_partition = random.choice(list(filter(lambda x: x["owner_id"] == most_frequent_owner_id,
-                                                   ownership_list)))
+                to_steal_partition = random.choice(active_ownership_by_owner[most_frequent_owner_id])
                 to_steal_partition["owner_id"] = self.owner_id
                 to_claim.append(to_steal_partition)
         return to_claim
-
-    async def _claim_ownership(self, ownership_list):
-        if ownership_list:
-            claimed_list = await self.partition_manager.claim_ownership(ownership_list)
-            return claimed_list
-        else:
-            return None
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
index 2714f675b28c..f9d9e46e7ed8 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
@@ -13,7 +13,8 @@ class CheckpointManager(object):
     The interaction with the chosen storage service is done via ~azure.eventhub.eventprocessor.PartitionManager.
 
     """
-    def __init__(self, partition_id: str, eventhub_name: str, consumer_group_name: str, owner_id: str, partition_manager: PartitionManager):
+    def __init__(self, partition_id: str, eventhub_name: str, consumer_group_name: str,
+                 owner_id: str, partition_manager: PartitionManager):
         self.partition_id = partition_id
         self.eventhub_name = eventhub_name
         self.consumer_group_name = consumer_group_name
@@ -22,11 +23,13 @@ def __init__(self, partition_id: str, eventhub_name: str, consumer_group_name: s
 
     async def update_checkpoint(self, offset, sequence_number=None):
         """
-        Updates the checkpoint using the given information for the associated partition and consumer group in the chosen storage service.
+        Updates the checkpoint using the given information for the associated partition and consumer group in the
+        chosen storage service.
 
         :param offset: The offset of the ~azure.eventhub.EventData the new checkpoint will be associated with.
         :type offset: str
-        :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be associated with.
+        :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be
+        associated with.
         :type sequence_number: int
         :return: None
         """
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index 23178deba76a..dfed927015c1 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -53,9 +53,11 @@ async def process_events(self, events, checkpoint_manager):
                 await partition_manager.close()
 
     """
-    def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
-                 partition_processor_factory: Callable[..., PartitionProcessor],
-                 partition_manager: PartitionManager, **kwargs):
+    def __init__(
+            self, eventhub_client: EventHubClient, consumer_group_name: str,
+            partition_processor_factory: Callable[..., PartitionProcessor],
+            partition_manager: PartitionManager, **kwargs
+    ):
         """
         Instantiate an EventProcessor.
 
@@ -88,7 +90,8 @@ def __init__(self, eventhub_client: EventHubClient, consumer_group_name: str,
         self._max_batch_size = eventhub_client.config.max_batch_size
         self._receive_timeout = eventhub_client.config.receive_timeout
         self._polling_interval = kwargs.get("polling_interval", 10)
-        self._ownership_timeout = self._polling_interval * 2  # TODO: Team haven't decided if this is a separate argument
+        self._ownership_timeout = self._polling_interval * 2
+        # TODO: Team haven't decided if this is a separate argument
         self._tasks = {}  # type: Dict[str, asyncio.Task]
         self._id = str(uuid.uuid4())
         self._running = False
@@ -104,13 +107,15 @@ async def start(self):
         2. Cancels tasks for partitions that are no longer owned by this EventProcessor
         3. Creates tasks for partitions that are newly claimed by this EventProcessor
         4. Keeps tasks running for partitions that haven't changed ownership
-        5. Each task repeatedly calls EvenHubConsumer.receive() to retrieve events and call user defined partition processor
+        5. Each task repeatedly calls EvenHubConsumer.receive() to retrieve events and
+        call user defined partition processor
 
         :return: None
 
         """
         log.info("EventProcessor %r is being started", self._id)
-        ownership_manager = OwnershipManager(self, self._eventhub_client, self._ownership_timeout)
+        ownership_manager = OwnershipManager(self._eventhub_client, self._consumer_group_name, self._id,
+                                             self._partition_manager, self._ownership_timeout)
         if not self._running:
             self._running = True
             while self._running:
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py
index 368cd8469f10..1d8add0f49a0 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py
@@ -10,7 +10,7 @@ def get_running_loop():
     try:
         return asyncio.get_running_loop()
     except AttributeError:  # 3.5 / 3.6
-        loop = asyncio._get_running_loop()  # pylint: disable=protected-access
+        loop = asyncio._get_running_loop()  # pylint: disable=protected-access, no-member
         if loop is None:
             raise RuntimeError('No running event loop')
         return loop

From c126bea5d4053c7c720dea12e8826e0f36f0a5e5 Mon Sep 17 00:00:00 2001
From: Azure SDK for Python bot <aspysdk2@microsoft.com>
Date: Fri, 30 Aug 2019 00:06:32 +0000
Subject: [PATCH 14/47] Packaging update of azure-mgmt-datalake-analytics

---
 .../azure-mgmt-datalake-analytics/MANIFEST.in |  5 +++++
 .../azure-mgmt-datalake-analytics/README.rst  | 21 +------------------
 .../azure-mgmt-datalake-analytics/setup.py    |  2 +-
 3 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
index bb37a2723dae..c0cf549166ee 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
@@ -1 +1,6 @@
+recursive-include tests *.py *.yaml
 include *.rst
+include azure/__init__.py
+include azure/mgmt/__init__.py
+include azure/mgmt/datalake/__init__.py
+
diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/README.rst b/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
index 756ded7f07a9..fb18ab8d347d 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
@@ -6,7 +6,7 @@ This is the Microsoft Azure Data Lake Analytics Management Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
@@ -14,25 +14,6 @@ For the older Azure Service Management (ASM) libraries, see
 For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
 
 
-Compatibility
-=============
-
-**IMPORTANT**: If you have an earlier version of the azure package
-(version < 1.0), you should uninstall it before installing this package.
-
-You can check the version using pip:
-
-.. code:: shell
-
-    pip freeze
-
-If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
-
-.. code:: shell
-
-    pip uninstall azure
-
-
 Usage
 =====
 
diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/setup.py b/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
index 68f340388091..4ba2e4e83639 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
@@ -53,6 +53,7 @@
     version=version,
     description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
     long_description=readme + '\n\n' + history,
+    long_description_content_type='text/x-rst',
     license='MIT License',
     author='Microsoft Corporation',
     author_email='azpysdkhelp@microsoft.com',
@@ -63,7 +64,6 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

From 40c7f03b3940069479340effe10afde00c41cbb4 Mon Sep 17 00:00:00 2001
From: Azure SDK for Python bot <aspysdk2@microsoft.com>
Date: Fri, 30 Aug 2019 00:06:33 +0000
Subject: [PATCH 15/47] Packaging update of azure-loganalytics

---
 .../azure-loganalytics/MANIFEST.in            |  3 ++-
 .../azure-loganalytics/README.rst             | 21 +------------------
 sdk/loganalytics/azure-loganalytics/setup.py  |  3 ++-
 3 files changed, 5 insertions(+), 22 deletions(-)

diff --git a/sdk/loganalytics/azure-loganalytics/MANIFEST.in b/sdk/loganalytics/azure-loganalytics/MANIFEST.in
index 88830671bc4d..74c0ac7e42a1 100644
--- a/sdk/loganalytics/azure-loganalytics/MANIFEST.in
+++ b/sdk/loganalytics/azure-loganalytics/MANIFEST.in
@@ -1,3 +1,4 @@
+recursive-include tests *.py *.yaml
 include *.rst
+include azure/__init__.py
 
-include azure/__init__.py
\ No newline at end of file
diff --git a/sdk/loganalytics/azure-loganalytics/README.rst b/sdk/loganalytics/azure-loganalytics/README.rst
index f2da77db1eca..78bf6021ad9e 100644
--- a/sdk/loganalytics/azure-loganalytics/README.rst
+++ b/sdk/loganalytics/azure-loganalytics/README.rst
@@ -3,30 +3,11 @@ Microsoft Azure SDK for Python
 
 This is the Microsoft Azure Log Analytics Client Library.
 
-This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
 
 For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
 
 
-Compatibility
-=============
-
-**IMPORTANT**: If you have an earlier version of the azure package
-(version < 1.0), you should uninstall it before installing this package.
-
-You can check the version using pip:
-
-.. code:: shell
-
-    pip freeze
-
-If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
-
-.. code:: shell
-
-    pip uninstall azure
-
-
 Usage
 =====
 
diff --git a/sdk/loganalytics/azure-loganalytics/setup.py b/sdk/loganalytics/azure-loganalytics/setup.py
index 87ff80807710..52e41c6df814 100644
--- a/sdk/loganalytics/azure-loganalytics/setup.py
+++ b/sdk/loganalytics/azure-loganalytics/setup.py
@@ -53,6 +53,7 @@
     version=version,
     description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
     long_description=readme + '\n\n' + history,
+    long_description_content_type='text/x-rst',
     license='MIT License',
     author='Microsoft Corporation',
     author_email='azpysdkhelp@microsoft.com',
@@ -63,7 +64,6 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -77,6 +77,7 @@
     ]),
     install_requires=[
         'msrest>=0.5.0',
+        'msrestazure>=0.4.32,<2.0.0',
         'azure-common~=1.1',
     ],
     extras_require={

From cf22c7c2e6f83d4e31a741c190ab63e19d417cc2 Mon Sep 17 00:00:00 2001
From: Azure SDK for Python bot <aspysdk2@microsoft.com>
Date: Fri, 30 Aug 2019 00:06:34 +0000
Subject: [PATCH 16/47] Packaging update of azure-mgmt-storage

---
 sdk/storage/azure-mgmt-storage/README.rst | 2 +-
 sdk/storage/azure-mgmt-storage/setup.py   | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/sdk/storage/azure-mgmt-storage/README.rst b/sdk/storage/azure-mgmt-storage/README.rst
index e8a573239aa7..5b9e52dc5b2d 100644
--- a/sdk/storage/azure-mgmt-storage/README.rst
+++ b/sdk/storage/azure-mgmt-storage/README.rst
@@ -6,7 +6,7 @@ This is the Microsoft Azure Storage Management Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
diff --git a/sdk/storage/azure-mgmt-storage/setup.py b/sdk/storage/azure-mgmt-storage/setup.py
index 8bef269c261e..8609a1e171a7 100644
--- a/sdk/storage/azure-mgmt-storage/setup.py
+++ b/sdk/storage/azure-mgmt-storage/setup.py
@@ -64,7 +64,6 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

From fa804f4d087823e82e118358578d4050fa9c8b40 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Fri, 30 Aug 2019 09:18:25 -0700
Subject: [PATCH 17/47] code review fixes and pylint error

---
 .../azure/eventhub/eventprocessor/__init__.py |  4 +-
 .../eventprocessor/_ownership_manager.py      | 27 ++++----
 .../eventprocessor/checkpoint_manager.py      |  4 +-
 .../eventprocessor/event_processor.py         | 68 ++++++++-----------
 .../eventprocessor/partition_manager.py       | 11 +--
 .../eventprocessor/partition_processor.py     | 19 ++++--
 .../sqlite3_partition_manager.py              | 20 +++---
 .../eventprocessor/event_processor_example.py |  4 +-
 8 files changed, 80 insertions(+), 77 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
index f4b48afac6f3..6280a03fce12 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
@@ -4,6 +4,7 @@
 # -----------------------------------------------------------------------------------
 
 from .event_processor import EventProcessor
+from .checkpoint_manager import CheckpointManager
 from .partition_processor import PartitionProcessor, CloseReason
 from .partition_manager import PartitionManager
 from .sqlite3_partition_manager import Sqlite3PartitionManager
@@ -13,5 +14,6 @@
     'EventProcessor',
     'PartitionProcessor',
     'PartitionManager',
+    'CheckpointManager',
     'Sqlite3PartitionManager',
-]
\ No newline at end of file
+]
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
index 5a49a1d093e9..e0c3d213c63a 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -6,8 +6,8 @@
 import time
 import random
 import math
-import collections
-from collections import Counter
+from typing import List
+from collections import Counter, defaultdict
 from azure.eventhub.aio import EventHubClient
 from .partition_manager import PartitionManager
 
@@ -26,7 +26,7 @@ def __init__(
             self, eventhub_client: EventHubClient, consumer_group_name: str, owner_id: str,
             partition_manager: PartitionManager, ownership_timeout: int
     ):
-        self.cached_parition_ids = []
+        self.cached_parition_ids = []  # type: List[str]
         self.eventhub_client = eventhub_client
         self.eventhub_name = eventhub_client.eh_name
         self.consumer_group_name = consumer_group_name
@@ -45,7 +45,7 @@ async def claim_ownership(self):
         """
         if not self.cached_parition_ids:
             await self._retrieve_partition_ids()
-        to_claim = await self._balance_ownership()
+        to_claim = await self._balance_ownership(self.cached_parition_ids)
         claimed_list = await self.partition_manager.claim_ownership(to_claim) if to_claim else None
         return claimed_list
 
@@ -56,7 +56,7 @@ async def _retrieve_partition_ids(self):
         """
         self.cached_parition_ids = await self.eventhub_client.get_partition_ids()
 
-    async def _balance_ownership(self):
+    async def _balance_ownership(self, all_partition_ids):
         """Balances and claims ownership of partitions for this EventProcessor.
         The balancing algorithm is:
         1. Find partitions with inactive ownership and partitions that haven never been claimed before
@@ -87,19 +87,19 @@ async def _balance_ownership(self):
         )
         now = time.time()
         ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
-        not_owned_partition_ids = [pid for pid in self.cached_parition_ids if pid not in ownership_dict]
+        not_owned_partition_ids = [pid for pid in all_partition_ids if pid not in ownership_dict]
         timed_out_partition_ids = [ownership["partition_id"] for ownership in ownership_list
                                    if ownership["last_modified_time"] + self.ownership_timeout < now]
         claimable_partition_ids = not_owned_partition_ids + timed_out_partition_ids
         active_ownership = [ownership for ownership in ownership_list
                             if ownership["last_modified_time"] + self.ownership_timeout >= now]
-        active_ownership_by_owner = collections.defaultdict(list)
+        active_ownership_by_owner = defaultdict(list)
         for ownership in active_ownership:
             active_ownership_by_owner[ownership["owner_id"]].append(ownership)
         active_ownership_self = active_ownership_by_owner[self.owner_id]
 
         # calculate expected count per owner
-        all_partition_count = len(self.cached_parition_ids)
+        all_partition_count = len(all_partition_ids)
         owners_count = len(active_ownership_by_owner) + \
                        (0 if self.owner_id in active_ownership_by_owner else 1)
         expected_count_per_owner = all_partition_count // owners_count
@@ -109,20 +109,21 @@ async def _balance_ownership(self):
         to_claim = active_ownership_self
         if len(active_ownership_self) > most_count_allowed_per_owner:  # needs to abandon a partition
             to_claim.pop()  # abandon one partition if owned too many
-            # TODO: Release a ownership immediately so other EventProcessors won't need to wait it to timeout
-        elif len(active_ownership_self) < expected_count_per_owner:  # Either claims an inactive partition, or steals from other owners
+            # TODO: Release an ownership immediately so other EventProcessors won't need to wait it to timeout
+        elif len(active_ownership_self) < expected_count_per_owner:
+            # Either claims an inactive partition, or steals from other owners
             if claimable_partition_ids:  # claim an inactive partition if there is
                 random_partition_id = random.choice(claimable_partition_ids)
                 random_chosen_to_claim = ownership_dict.get(random_partition_id,
                                                             {"partition_id": random_partition_id,
                                                              "eventhub_name": self.eventhub_client.eh_name,
-                                                             "consumer_group_name": self.consumer_group_name,
-                                                             "owner_level": 0})  # TODO: consider removing owner_level
+                                                             "consumer_group_name": self.consumer_group_name
+                                                             })
                 random_chosen_to_claim["owner_id"] = self.owner_id
                 to_claim.append(random_chosen_to_claim)
             else:  # steal from another owner that has the most count
                 active_ownership_count_group_by_owner = Counter(
-                    (x, len(y)) for x, y in active_ownership_by_owner.items())
+                    dict((x, len(y)) for x, y in active_ownership_by_owner.items()))
                 most_frequent_owner_id = active_ownership_count_group_by_owner.most_common(1)[0][0]
                 # randomly choose a partition to steal from the most_frequent_owner
                 to_steal_partition = random.choice(active_ownership_by_owner[most_frequent_owner_id])
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
index f9d9e46e7ed8..3be0c5d945c3 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
@@ -13,8 +13,8 @@ class CheckpointManager(object):
     The interaction with the chosen storage service is done via ~azure.eventhub.eventprocessor.PartitionManager.
 
     """
-    def __init__(self, partition_id: str, eventhub_name: str, consumer_group_name: str,
-                 owner_id: str, partition_manager: PartitionManager):
+    def __init__(self, eventhub_name: str, consumer_group_name: str,
+                 partition_id: str, owner_id: str, partition_manager: PartitionManager):
         self.partition_id = partition_id
         self.eventhub_name = eventhub_name
         self.consumer_group_name = consumer_group_name
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index dfed927015c1..b300b7bd0a1c 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -20,7 +20,7 @@
 OWNER_LEVEL = 0
 
 
-class EventProcessor(object):
+class EventProcessor(object):  # pylint:disable=too-many-instance-attributes
     """
     An EventProcessor constantly receives events from all partitions of the Event Hub in the context of a given
     consumer group. The received data will be sent to PartitionProcessor to be processed.
@@ -163,72 +163,62 @@ def _create_tasks_for_claimed_ownership(self, to_claim_ownership_list):
     async def _receive(self, ownership):
         log.info("start ownership, %r", ownership)
         partition_processor = self._partition_processor_factory()
-        if not hasattr(partition_processor, "process_events"):
-            log.error(
-                "Fatal error: a partition processor should at least have method process_events(events, checkpoint_manager). EventProcessor will stop.")
-            await self.stop()
-            raise TypeError("Partition processor must has method process_events(events, checkpoint_manager")
-
-        partition_consumer = self._eventhub_client.create_consumer(
-            ownership["consumer_group_name"],
-            ownership["partition_id"],
-            EventPosition(ownership.get("offset", self._initial_event_position))
-        )
+        partition_id = ownership["partition_id"]
+        eventhub_name = ownership["eventhub_name"]
+        consumer_group_name = ownership["consumer_group_name"]
+        owner_id = ownership["owner_id"]
         checkpoint_manager = CheckpointManager(
-            ownership["partition_id"],
-            ownership["eventhub_name"],
-            ownership["consumer_group_name"],
-            ownership["owner_id"],
+            eventhub_name,
+            consumer_group_name,
+            partition_id,
+            owner_id,
             self._partition_manager
         )
-
-        async def initialize():
-            if hasattr(partition_processor, "initialize"):
-                await partition_processor.initialize()
-
-        async def process_error(err):
-            if hasattr(partition_processor, "process_error"):
-                await partition_processor.process_error(err, checkpoint_manager)
-
-        async def close(close_reason):
-            if hasattr(partition_processor, "close"):
-                await partition_processor.close(close_reason, checkpoint_manager)
+        partition_processor.eventhub_name = ownership
+        partition_processor.checkpoint_manager = checkpoint_manager
+        partition_consumer = self._eventhub_client.create_consumer(
+            consumer_group_name,
+            partition_id,
+            EventPosition(ownership.get("offset", self._initial_event_position))
+        )
 
         try:
             while True:
                 try:
-                    await initialize()
+                    await partition_processor.initialize()
                     events = await partition_consumer.receive(timeout=self._receive_timeout)
                     await partition_processor.process_events(events, checkpoint_manager)
                 except asyncio.CancelledError as cancelled_error:
                     log.info(
-                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                        "is cancelled",
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                        " is cancelled",
                         ownership["owner_id"],
                         ownership["eventhub_name"],
                         ownership["partition_id"],
                         ownership["consumer_group_name"]
                     )
-                    await process_error(cancelled_error)
-                    await close(CloseReason.SHUTDOWN)
+                    await partition_processor.process_error(cancelled_error, checkpoint_manager)
+                    await partition_processor.close(CloseReason.SHUTDOWN, checkpoint_manager)
                     # TODO: release the ownership immediately via partition manager
                     break
                 except EventHubError as eh_err:
-                    reason = CloseReason.OWNERSHIP_LOST if eh_err.error == "link:stolen" else CloseReason.EVENTHUB_EXCEPTION
+                    reason = CloseReason.OWNERSHIP_LOST if eh_err.error == "link:stolen" \
+                        else CloseReason.EVENTHUB_EXCEPTION
                     log.warning(
-                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r "
-                        "has met an exception receiving events. It's being closed. The exception is %r.",
+                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                        " has met an exception receiving events. It's being closed. The exception is %r.",
                         ownership["owner_id"],
                         ownership["eventhub_name"],
                         ownership["partition_id"],
                         ownership["consumer_group_name"],
                         eh_err
                     )
-                    await process_error(eh_err)
-                    await close(reason)  # An EventProcessor will pick up this partition again after the ownership is released
+                    await partition_processor.process_error(eh_err, checkpoint_manager)
+                    await partition_processor.close(reason, checkpoint_manager)
+                    # An EventProcessor will pick up this partition again after the ownership is released
                     # TODO: release the ownership immediately via partition manager
                     break
-                except Exception as exp:
+                except Exception as exp:  # pylint:disable=broad-except
                     log.warning(exp)
         finally:
             await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
index e4ecb1bec824..e6d3e5df2538 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
@@ -34,7 +34,7 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
                 last_modified_time
                 etag
         """
-        pass
+
 
     @abstractmethod
     async def claim_ownership(self, partitions: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
@@ -54,13 +54,14 @@ async def claim_ownership(self, partitions: Iterable[Dict[str, Any]]) -> Iterabl
                 last_modified_time
                 etag
         """
-        pass
+
 
     @abstractmethod
     async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
                                 offset, sequence_number) -> None:
         """
-        Updates the checkpoint using the given information for the associated partition and consumer group in the chosen storage service.
+        Updates the checkpoint using the given information for the associated partition and
+        consumer group in the chosen storage service.
 
         :param eventhub_name: The name of the specific Event Hub the ownership are associated with, relative to
          the Event Hubs namespace that contains it.
@@ -73,11 +74,11 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
         :type owner_id: str
         :param offset: The offset of the ~azure.eventhub.EventData the new checkpoint will be associated with.
         :type offset: str
-        :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint will be associated with.
+        :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint
+        will be associated with.
         :type sequence_number: int
         :return:
         """
-        pass
 
     async def close(self):
         pass
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 60a95b99e628..37be7006e244 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -4,11 +4,10 @@
 # -----------------------------------------------------------------------------------
 
 from typing import List
-from typing_extensions import Protocol
+from abc import ABC
 from enum import Enum
-from .checkpoint_manager import CheckpointManager
-
 from azure.eventhub import EventData
+from .checkpoint_manager import CheckpointManager
 
 
 class CloseReason(Enum):
@@ -17,12 +16,14 @@ class CloseReason(Enum):
     EVENTHUB_EXCEPTION = 2  # Exception happens during receiving events
 
 
-class PartitionProcessor(Protocol):
+class PartitionProcessor(ABC):
     """
     PartitionProcessor processes events received from the Azure Event Hubs service. A single instance of a class
-    implementing this abstract class will be created for every partition the associated ~azure.eventhub.eventprocessor.EventProcessor owns.
+    implementing this abstract class will be created for every partition the associated
+    ~azure.eventhub.eventprocessor.EventProcessor owns.
 
     """
+
     async def initialize(self):
         pass
 
@@ -34,15 +35,18 @@ async def close(self, reason, checkpoint_manager: CheckpointManager):
 
         :param reason: Reason for closing the PartitionProcessor.
         :type reason: ~azure.eventhub.eventprocessor.CloseReason
+        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
+        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
 
         """
-        pass
 
     async def process_events(self, events: List[EventData], checkpoint_manager: CheckpointManager):
         """Called when a batch of events have been received.
 
         :param events: Received events.
         :type events: list[~azure.eventhub.common.EventData]
+        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
+        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
 
         """
         raise NotImplementedError
@@ -52,6 +56,7 @@ async def process_error(self, error, checkpoint_manager: CheckpointManager):
 
         :param error: The error that happens.
         :type error: Exception
+        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
+        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
 
         """
-        pass
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index bf4c46c95eb9..50a7d8caf29e 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -53,9 +53,6 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
                   + ",".join(self.primary_keys)\
                   + "))"
             c.execute(sql)
-        except sqlite3.OperationalError:
-            raise
-            pass
         finally:
             c.close()
         self.conn = conn
@@ -100,13 +97,16 @@ async def claim_ownership(self, partitions):
                     if p.get("etag") == cursor_fetch[0][0]:
                         p["last_modified_time"] = time.time()
                         p["etag"] = str(uuid.uuid4())
-                        other_fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.other_fields))
+                        other_fields_without_checkpoint = list(
+                            filter(lambda x: x not in self.checkpoint_fields, self.other_fields)
+                        )
                         sql = "update " + _check_table_name(self.ownership_table) + " set "\
                                        + ','.join([field+"=?" for field in other_fields_without_checkpoint])\
                                        + " where "\
                                        + " and ".join([field+"=?" for field in self.primary_keys])
 
-                        cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint) + tuple(p.get(field) for field in self.primary_keys))
+                        cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint)
+                                       + tuple(p.get(field) for field in self.primary_keys))
                         result.append(p)
                     else:
                         logger.info("EventProcessor %r failed to claim partition %r "
@@ -121,15 +121,19 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             offset, sequence_number):
         cursor = self.conn.cursor()
         try:
-            cursor.execute("select owner_id from " + _check_table_name(self.ownership_table) + " where eventhub_name=? and consumer_group_name=? and partition_id=?",
+            cursor.execute("select owner_id from " + _check_table_name(self.ownership_table)
+                           + " where eventhub_name=? and consumer_group_name=? and partition_id=?",
                            (eventhub_name, consumer_group_name, partition_id))
             cursor_fetch = cursor.fetchall()
             if cursor_fetch and owner_id == cursor_fetch[0][0]:
-                cursor.execute("update " + _check_table_name(self.ownership_table) + " set offset=?, sequence_number=? where eventhub_name=? and consumer_group_name=? and partition_id=?",
+                cursor.execute("update " + _check_table_name(self.ownership_table)
+                               + " set offset=?, sequence_number=? "
+                                 "where eventhub_name=? and consumer_group_name=? and partition_id=?",
                                (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
                 self.conn.commit()
             else:
-                logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership", partition_id)
+                logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership",
+                            partition_id)
 
         finally:
             cursor.close()
diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
index a2165eb81110..c17bb95bd4cf 100644
--- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
+++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
@@ -2,7 +2,7 @@
 import logging
 import os
 from azure.eventhub.aio import EventHubClient
-from azure.eventhub.eventprocessor import EventProcessor
+from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
 from azure.eventhub.eventprocessor import Sqlite3PartitionManager
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
@@ -17,7 +17,7 @@ async def do_operation(event):
     print(event)
 
 
-class MyPartitionProcessor(object):
+class MyPartitionProcessor(PartitionProcessor):
     async def process_events(self, events, checkpoint_manager):
         if events:
             await asyncio.gather(*[do_operation(event) for event in events])

From e5f3b5072f790a69379b05988d3f4738ad26a30e Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Fri, 30 Aug 2019 10:26:57 -0700
Subject: [PATCH 18/47] reduce dictionary access

---
 .../eventhub/eventprocessor/event_processor.py   | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index b300b7bd0a1c..aaeb2de92dd0 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -192,10 +192,10 @@ async def _receive(self, ownership):
                     log.info(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
                         " is cancelled",
-                        ownership["owner_id"],
-                        ownership["eventhub_name"],
-                        ownership["partition_id"],
-                        ownership["consumer_group_name"]
+                        owner_id,
+                        eventhub_name,
+                        partition_id,
+                        consumer_group_name
                     )
                     await partition_processor.process_error(cancelled_error, checkpoint_manager)
                     await partition_processor.close(CloseReason.SHUTDOWN, checkpoint_manager)
@@ -207,10 +207,10 @@ async def _receive(self, ownership):
                     log.warning(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
                         " has met an exception receiving events. It's being closed. The exception is %r.",
-                        ownership["owner_id"],
-                        ownership["eventhub_name"],
-                        ownership["partition_id"],
-                        ownership["consumer_group_name"],
+                        owner_id,
+                        eventhub_name,
+                        partition_id,
+                        consumer_group_name,
                         eh_err
                     )
                     await partition_processor.process_error(eh_err, checkpoint_manager)

From 27cb0bf4cbce41fa45d64c9d771ffc6fa557b8f8 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 28 Aug 2019 13:42:32 -0700
Subject: [PATCH 19/47] initial blob storage

---
 .../HISTORY.md                                |  0
 .../LICENSE                                   | 21 +++++
 .../MANIFEST.in                               |  2 +
 .../README.md                                 |  0
 .../azure/__init__.py                         |  0
 .../azure/ehplugin/__init__.py                |  0
 .../ehplugin/partitionmanager/__init__.py     |  0
 .../partitionmanager/blobstorage/__init__.py  |  6 ++
 .../blobstorage/blobstoragepm.py              | 80 +++++++++++++++++++
 .../conftest.py                               |  0
 .../dev_requirements.txt                      |  1 +
 .../event_processor_blob_storage_example.py   | 42 ++++++++++
 .../sdk_packaging.toml                        |  2 +
 .../setup.cfg                                 |  2 +
 .../setup.py                                  | 80 +++++++++++++++++++
 15 files changed, 236 insertions(+)
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
 create mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
new file mode 100644
index 000000000000..21071075c245
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
new file mode 100644
index 000000000000..50c61fef797b
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.md
+include azure/__init__.py
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
new file mode 100644
index 000000000000..3057ad223050
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
@@ -0,0 +1,6 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+__version__ = "1.0.0b1"
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
new file mode 100644
index 000000000000..f0e0230be351
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -0,0 +1,80 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+from typing import Iterable, Dict, Any
+import logging
+from collections import defaultdict
+import asyncio
+
+from azure.eventhub.eventprocessor import PartitionManager
+from azure.storage.blob.aio import ContainerClient
+
+logger = logging.getLogger(__name__)
+UPLOAD_DATA = ""
+
+
+class BlobPartitionManager(PartitionManager):
+    def __init__(self, container_client: ContainerClient):
+        self._container_client = container_client
+        # self._ownership_cache = {}
+        # self._ownership_locks = defaultdict(asyncio.Lock)
+
+    async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
+        blobs = self._container_client.list_blobs(include=['metadata'])
+        result = []
+        async for b in blobs:
+            metadata = b.metadata
+            ownership = {
+                "eventhub_name": eventhub_name,
+                "consumer_group_name": consumer_group_name,
+                "partition_id": b.name,
+                "etag": b.etag,
+                "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
+            }
+            ownership.update(metadata)
+            result.append(ownership)
+        return result
+
+    async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
+        result = []
+        for ownership in ownership_list:
+            metadata = {"owner_id": ownership["owner_id"]}
+            if "offset" in ownership:
+                metadata["offset"] = ownership["offset"]
+            if "sequence_number" in ownership:
+                metadata["sequence_number"] = ownership["sequence_number"]
+            name = ownership["partition_id"]
+            try:
+                etag = ownership.get("etag")
+                if etag:
+                    etag_match = {"if_match": '"'+etag+'"'}
+                else:
+                    etag_match = {"if_none_match": '"*"'}
+                blob_client = await self._container_client.upload_blob(
+                    name=name, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
+                )
+
+                uploaded_blob_properties = await blob_client.get_blob_properties()
+                ownership["etag"] = uploaded_blob_properties.etag
+                ownership["last_modified_time"] = uploaded_blob_properties.last_modified
+            except Exception as err:
+                logger.info("Claim error occurred: %r", err)
+                raise
+            result.append(ownership)
+        return result
+
+    async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
+                                offset, sequence_number) -> None:
+
+        metadata = {
+            "owner_id": owner_id,
+            "offset": offset,
+            "sequence_number": sequence_number
+        }
+        try:
+            blob_client = await self._container_client.upload_blob(name=partition_id, data=UPLOAD_DATA, overwrite=True)
+
+        except Exception as err:
+            logger.info("Checkpoint error occurred: %r", err)
+            raise
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
new file mode 100644
index 000000000000..092dbcdb7de7
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
@@ -0,0 +1 @@
+-e ../../eventhub/azure-eventhubs
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
new file mode 100644
index 000000000000..519a3c468460
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -0,0 +1,42 @@
+import asyncio
+import logging
+import os
+from azure.eventhub.aio import EventHubClient
+from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
+from azure.ehplugin.partitionmanager.blobstorage.blobstoragepm import BlobPartitionManager
+from azure.storage.blob.aio import ContainerClient
+
+RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
+RETRY_TOTAL = 3  # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small
+CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
+STORAGE_CONNECTION_STR = "DefaultEndpointsProtocol=https;AccountName=ephstg;AccountKey=JX+/0R9IFtbo406MlvgLb4/ihT66kZh7vyeqUoDZkS5RKeUi7CBO+3mpl1gnNTf2Do6uUFjDQtdr1i2idcQ7Uw==;BlobEndpoint=https://ephstg.blob.core.windows.net/;QueueEndpoint=https://ephstg.queue.core.windows.net/;TableEndpoint=https://ephstg.table.core.windows.net/;FileEndpoint=https://ephstg.file.core.windows.net/;"
+
+logging.basicConfig(level=logging.INFO)
+
+
+async def do_operation(event):
+    # do some sync or async operations. If the operation is i/o intensive, async will have better performance
+    print(event)
+
+
+class MyPartitionProcessor(PartitionProcessor):
+    async def process_events(self, events, checkpoint_manager):
+        if events:
+            await asyncio.gather(*[do_operation(event) for event in events])
+            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+        else:
+            print("empty events received", "partition:", checkpoint_manager.partition_id)
+
+
+if __name__ == '__main__':
+    loop = asyncio.get_event_loop()
+    client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
+    container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, container="eventprocessor")
+    partition_manager = BlobPartitionManager(container_client=container_client)
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
+    try:
+        loop.run_until_complete(event_processor.start())
+    except KeyboardInterrupt:
+        loop.run_until_complete(event_processor.stop())
+    finally:
+        loop.stop()
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
new file mode 100644
index 000000000000..e7687fdae93b
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
@@ -0,0 +1,2 @@
+[packaging]
+auto_update = false
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
new file mode 100644
index 000000000000..3480374bc2f2
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal=1
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
new file mode 100644
index 000000000000..78e798359dfe
--- /dev/null
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+#--------------------------------------------------------------------------
+
+import re
+import os.path
+import sys
+from io import open
+from setuptools import find_packages, setup
+
+
+# Change the PACKAGE_NAME only to change folder and different name
+PACKAGE_NAME = "azure-ehplugin-partitionmanager-blobstorage"
+PACKAGE_PPRINT_NAME = "Event Hubs Event Procesor Partition Manager implementation with Blob Storage"
+
+# a-b-c => a/b/c
+package_folder_path = PACKAGE_NAME.replace('-', '/')
+# a-b-c => a.b.c
+namespace_name = PACKAGE_NAME.replace('-', '.')
+
+# Version extraction inspired from 'requests'
+with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd:
+    version = re.search(r'^__version__\s*=\s*[\'"]([^\'"]*)[\'"]',
+                        fd.read(), re.MULTILINE).group(1)
+
+if not version:
+    raise RuntimeError('Cannot find version information')
+
+with open('README.md') as f:
+    readme = f.read()
+with open('HISTORY.md') as f:
+    history = f.read()
+
+exclude_packages = [
+        'tests',
+        'examples',
+        # Exclude packages that will be covered by PEP420 or nspkg
+        'azure',
+    ]
+
+if sys.version_info < (3, 5, 3):
+    exclude_packages.extend([
+        '*.aio',
+        '*.aio.*',
+    ])
+
+setup(
+    name=PACKAGE_NAME,
+    version=version,
+    description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
+    long_description=readme + '\n\n' + history,
+    long_description_content_type='text/markdown',
+    license='MIT License',
+    author='Microsoft Corporation',
+    author_email='azpysdkhelp@microsoft.com',
+    url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'License :: OSI Approved :: MIT License',
+    ],
+    zip_safe=False,
+    packages=find_packages(exclude=exclude_packages),
+    install_requires=[
+
+    ],
+    extras_require={
+        ":python_version<'3.0'": ['azure-nspkg'],
+    }
+)

From f6d77e7fbdaf5a0ae49cf10e5c63700cf2ba72c8 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 28 Aug 2019 14:13:26 -0700
Subject: [PATCH 20/47] fix leaking connection str

---
 .../examples/event_processor_blob_storage_example.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
index 519a3c468460..a802bb2b3a64 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -9,7 +9,7 @@
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
 RETRY_TOTAL = 3  # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small
 CONNECTION_STR = os.environ["EVENT_HUB_CONN_STR"]
-STORAGE_CONNECTION_STR = "DefaultEndpointsProtocol=https;AccountName=ephstg;AccountKey=JX+/0R9IFtbo406MlvgLb4/ihT66kZh7vyeqUoDZkS5RKeUi7CBO+3mpl1gnNTf2Do6uUFjDQtdr1i2idcQ7Uw==;BlobEndpoint=https://ephstg.blob.core.windows.net/;QueueEndpoint=https://ephstg.queue.core.windows.net/;TableEndpoint=https://ephstg.table.core.windows.net/;FileEndpoint=https://ephstg.file.core.windows.net/;"
+STORAGE_CONNECTION_STR = os.environ["AZURE_STORAGE_CONN_STR"]
 
 logging.basicConfig(level=logging.INFO)
 

From da6199fb1b96167aa95382309b5da9f734eb1309 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 11:07:17 -0700
Subject: [PATCH 21/47] Change test polling to 5 sec

---
 .../examples/event_processor_blob_storage_example.py            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
index a802bb2b3a64..48c30689a4ea 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -33,7 +33,7 @@ async def process_events(self, events, checkpoint_manager):
     client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
     container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, container="eventprocessor")
     partition_manager = BlobPartitionManager(container_client=container_client)
-    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=5)
     try:
         loop.run_until_complete(event_processor.start())
     except KeyboardInterrupt:

From e6a7c5e1215a0314749599712bf83dbda552e7ae Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 11:07:47 -0700
Subject: [PATCH 22/47] Add async lock to ensure etag consistency

---
 .../blobstorage/blobstoragepm.py              | 106 ++++++++++--------
 1 file changed, 59 insertions(+), 47 deletions(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
index f0e0230be351..fba1f52c5b55 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -6,6 +6,7 @@
 import logging
 from collections import defaultdict
 import asyncio
+from azure.core.exceptions import ResourceModifiedError, ResourceExistsError
 
 from azure.eventhub.eventprocessor import PartitionManager
 from azure.storage.blob.aio import ContainerClient
@@ -17,51 +18,55 @@
 class BlobPartitionManager(PartitionManager):
     def __init__(self, container_client: ContainerClient):
         self._container_client = container_client
-        # self._ownership_cache = {}
-        # self._ownership_locks = defaultdict(asyncio.Lock)
+        self._cached_ownership_dict = defaultdict(dict)
+        self._lock = asyncio.Lock()
 
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
-        blobs = self._container_client.list_blobs(include=['metadata'])
-        result = []
-        async for b in blobs:
-            metadata = b.metadata
-            ownership = {
-                "eventhub_name": eventhub_name,
-                "consumer_group_name": consumer_group_name,
-                "partition_id": b.name,
-                "etag": b.etag,
-                "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
-            }
-            ownership.update(metadata)
-            result.append(ownership)
-        return result
+        async with self._lock:
+            blobs = self._container_client.list_blobs(include=['metadata'])
+            # result = []
+            async for b in blobs:
+                metadata = b.metadata
+                ownership = {
+                    "eventhub_name": eventhub_name,
+                    "consumer_group_name": consumer_group_name,
+                    "partition_id": b.name,
+                    "owner_id": metadata["owner_id"],
+                    "etag": b.etag,
+                    "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
+                }
+                ownership.update(metadata)
+                self._cached_ownership_dict[b.name] = ownership
+        return self._cached_ownership_dict.values()
 
     async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
         result = []
-        for ownership in ownership_list:
-            metadata = {"owner_id": ownership["owner_id"]}
-            if "offset" in ownership:
-                metadata["offset"] = ownership["offset"]
-            if "sequence_number" in ownership:
-                metadata["sequence_number"] = ownership["sequence_number"]
-            name = ownership["partition_id"]
-            try:
-                etag = ownership.get("etag")
-                if etag:
-                    etag_match = {"if_match": '"'+etag+'"'}
-                else:
-                    etag_match = {"if_none_match": '"*"'}
-                blob_client = await self._container_client.upload_blob(
-                    name=name, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
-                )
-
-                uploaded_blob_properties = await blob_client.get_blob_properties()
-                ownership["etag"] = uploaded_blob_properties.etag
-                ownership["last_modified_time"] = uploaded_blob_properties.last_modified
-            except Exception as err:
-                logger.info("Claim error occurred: %r", err)
-                raise
-            result.append(ownership)
+        async with self._lock:
+            for ownership in ownership_list:
+                metadata = {"owner_id": ownership["owner_id"]}
+                if "offset" in ownership:
+                    metadata["offset"] = ownership["offset"]
+                if "sequence_number" in ownership:
+                    metadata["sequence_number"] = ownership["sequence_number"]
+                partition_id = ownership["partition_id"]
+                try:
+                    etag = ownership.get("etag")
+                    if etag:
+                        etag_match = {"if_match": '"'+etag+'"'}
+                    else:
+                        etag_match = {"if_none_match": '"*"'}
+                    blob_client = await self._container_client.upload_blob(
+                        name=partition_id, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
+                    )
+                    uploaded_blob_properties = await blob_client.get_blob_properties()
+                    ownership["etag"] = uploaded_blob_properties.etag
+                    ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
+                    self._cached_ownership_dict[partition_id] = ownership
+                except (ResourceModifiedError, ResourceExistsError):
+                    logger.info("Partition %r was claimed by another EventProcessor", partition_id)
+                except Exception as err:
+                    logger.warning("Claim error occurred: %r", err)
+                result.append(ownership)
         return result
 
     async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
@@ -70,11 +75,18 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
         metadata = {
             "owner_id": owner_id,
             "offset": offset,
-            "sequence_number": sequence_number
+            "sequence_number": str(sequence_number)
         }
-        try:
-            blob_client = await self._container_client.upload_blob(name=partition_id, data=UPLOAD_DATA, overwrite=True)
-
-        except Exception as err:
-            logger.info("Checkpoint error occurred: %r", err)
-            raise
+        async with self._lock:
+            try:
+                blob_client = await self._container_client.upload_blob(
+                    name=partition_id, data=UPLOAD_DATA, metadata=metadata, overwrite=True)
+                uploaded_blob_properties = await blob_client.get_blob_properties()
+                cached_ownership = self._cached_ownership_dict[partition_id]
+                cached_ownership["etag"] = uploaded_blob_properties.etag
+                cached_ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
+            except (ResourceModifiedError, ResourceExistsError):
+                logger.info("Partition %r was claimed by another EventProcessor", partition_id)
+            except Exception as err:
+                logger.warning("Checkpoint error occurred: %r", err)
+                raise

From ebc4362dcd92d7849420c414cc461d6798756dd0 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 11:20:06 -0700
Subject: [PATCH 23/47] Add dependency to azure-storage

---
 .../azure-ehplugin-partitionmanager-blobstorage/setup.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
index 78e798359dfe..27e05164af51 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
@@ -72,7 +72,7 @@
     zip_safe=False,
     packages=find_packages(exclude=exclude_packages),
     install_requires=[
-
+        'azure-storage<13.0.0,>=12.0.0b2',
     ],
     extras_require={
         ":python_version<'3.0'": ['azure-nspkg'],

From 1503604ef030e91af072f49dce37bb6315f4efc4 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 11:20:30 -0700
Subject: [PATCH 24/47] Remove dependency on PartitionManager of azure-eventhub

---
 .../ehplugin/partitionmanager/blobstorage/blobstoragepm.py    | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
index fba1f52c5b55..e28e50873fec 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -7,15 +7,13 @@
 from collections import defaultdict
 import asyncio
 from azure.core.exceptions import ResourceModifiedError, ResourceExistsError
-
-from azure.eventhub.eventprocessor import PartitionManager
 from azure.storage.blob.aio import ContainerClient
 
 logger = logging.getLogger(__name__)
 UPLOAD_DATA = ""
 
 
-class BlobPartitionManager(PartitionManager):
+class BlobPartitionManager(object):
     def __init__(self, container_client: ContainerClient):
         self._container_client = container_client
         self._cached_ownership_dict = defaultdict(dict)

From c9707c4c870fb217f83164044b8806d0e7014743 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 11:56:06 -0700
Subject: [PATCH 25/47] Fix azure-storage-blob requirement error

---
 .../azure-ehplugin-partitionmanager-blobstorage/setup.py        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
index 27e05164af51..591569ab023c 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
@@ -72,7 +72,7 @@
     zip_safe=False,
     packages=find_packages(exclude=exclude_packages),
     install_requires=[
-        'azure-storage<13.0.0,>=12.0.0b2',
+        'azure-storage-blob<13.0.0,>=12.0.0b2',
     ],
     extras_require={
         ":python_version<'3.0'": ['azure-nspkg'],

From 8343876f993d67f8c8583c1f8648904561e7b80c Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 12:23:53 -0700
Subject: [PATCH 26/47] Revert "Packaging update of azure-mgmt-storage"

This reverts commit cf22c7c2e6f83d4e31a741c190ab63e19d417cc2.
---
 sdk/storage/azure-mgmt-storage/README.rst | 2 +-
 sdk/storage/azure-mgmt-storage/setup.py   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdk/storage/azure-mgmt-storage/README.rst b/sdk/storage/azure-mgmt-storage/README.rst
index 5b9e52dc5b2d..e8a573239aa7 100644
--- a/sdk/storage/azure-mgmt-storage/README.rst
+++ b/sdk/storage/azure-mgmt-storage/README.rst
@@ -6,7 +6,7 @@ This is the Microsoft Azure Storage Management Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
diff --git a/sdk/storage/azure-mgmt-storage/setup.py b/sdk/storage/azure-mgmt-storage/setup.py
index 8609a1e171a7..8bef269c261e 100644
--- a/sdk/storage/azure-mgmt-storage/setup.py
+++ b/sdk/storage/azure-mgmt-storage/setup.py
@@ -64,6 +64,7 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

From 66c5b31e2157c522bbc5396f1640f25d74074f59 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 12:25:58 -0700
Subject: [PATCH 27/47] Revert "Packaging update of azure-loganalytics"

This reverts commit 40c7f03b3940069479340effe10afde00c41cbb4.
---
 .../azure-loganalytics/MANIFEST.in            |  3 +--
 .../azure-loganalytics/README.rst             | 21 ++++++++++++++++++-
 sdk/loganalytics/azure-loganalytics/setup.py  |  3 +--
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/sdk/loganalytics/azure-loganalytics/MANIFEST.in b/sdk/loganalytics/azure-loganalytics/MANIFEST.in
index 74c0ac7e42a1..88830671bc4d 100644
--- a/sdk/loganalytics/azure-loganalytics/MANIFEST.in
+++ b/sdk/loganalytics/azure-loganalytics/MANIFEST.in
@@ -1,4 +1,3 @@
-recursive-include tests *.py *.yaml
 include *.rst
-include azure/__init__.py
 
+include azure/__init__.py
\ No newline at end of file
diff --git a/sdk/loganalytics/azure-loganalytics/README.rst b/sdk/loganalytics/azure-loganalytics/README.rst
index 78bf6021ad9e..f2da77db1eca 100644
--- a/sdk/loganalytics/azure-loganalytics/README.rst
+++ b/sdk/loganalytics/azure-loganalytics/README.rst
@@ -3,11 +3,30 @@ Microsoft Azure SDK for Python
 
 This is the Microsoft Azure Log Analytics Client Library.
 
-This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
 
 For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
 
 
+Compatibility
+=============
+
+**IMPORTANT**: If you have an earlier version of the azure package
+(version < 1.0), you should uninstall it before installing this package.
+
+You can check the version using pip:
+
+.. code:: shell
+
+    pip freeze
+
+If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
+
+.. code:: shell
+
+    pip uninstall azure
+
+
 Usage
 =====
 
diff --git a/sdk/loganalytics/azure-loganalytics/setup.py b/sdk/loganalytics/azure-loganalytics/setup.py
index 52e41c6df814..87ff80807710 100644
--- a/sdk/loganalytics/azure-loganalytics/setup.py
+++ b/sdk/loganalytics/azure-loganalytics/setup.py
@@ -53,7 +53,6 @@
     version=version,
     description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
     long_description=readme + '\n\n' + history,
-    long_description_content_type='text/x-rst',
     license='MIT License',
     author='Microsoft Corporation',
     author_email='azpysdkhelp@microsoft.com',
@@ -64,6 +63,7 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
@@ -77,7 +77,6 @@
     ]),
     install_requires=[
         'msrest>=0.5.0',
-        'msrestazure>=0.4.32,<2.0.0',
         'azure-common~=1.1',
     ],
     extras_require={

From bcd851a31259c9f9321ebf187663c1a504bd6a03 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 12:26:29 -0700
Subject: [PATCH 28/47] Revert "Packaging update of
 azure-mgmt-datalake-analytics"

This reverts commit c126bea5d4053c7c720dea12e8826e0f36f0a5e5.
---
 .../azure-mgmt-datalake-analytics/MANIFEST.in |  5 -----
 .../azure-mgmt-datalake-analytics/README.rst  | 21 ++++++++++++++++++-
 .../azure-mgmt-datalake-analytics/setup.py    |  2 +-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
index c0cf549166ee..bb37a2723dae 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/MANIFEST.in
@@ -1,6 +1 @@
-recursive-include tests *.py *.yaml
 include *.rst
-include azure/__init__.py
-include azure/mgmt/__init__.py
-include azure/mgmt/datalake/__init__.py
-
diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/README.rst b/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
index fb18ab8d347d..756ded7f07a9 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/README.rst
@@ -6,7 +6,7 @@ This is the Microsoft Azure Data Lake Analytics Management Client Library.
 Azure Resource Manager (ARM) is the next generation of management APIs that
 replace the old Azure Service Management (ASM).
 
-This package has been tested with Python 2.7, 3.5, 3.6 and 3.7.
+This package has been tested with Python 2.7, 3.4, 3.5, 3.6 and 3.7.
 
 For the older Azure Service Management (ASM) libraries, see
 `azure-servicemanagement-legacy <https://pypi.python.org/pypi/azure-servicemanagement-legacy>`__ library.
@@ -14,6 +14,25 @@ For the older Azure Service Management (ASM) libraries, see
 For a more complete set of Azure libraries, see the `azure <https://pypi.python.org/pypi/azure>`__ bundle package.
 
 
+Compatibility
+=============
+
+**IMPORTANT**: If you have an earlier version of the azure package
+(version < 1.0), you should uninstall it before installing this package.
+
+You can check the version using pip:
+
+.. code:: shell
+
+    pip freeze
+
+If you see azure==0.11.0 (or any version below 1.0), uninstall it first:
+
+.. code:: shell
+
+    pip uninstall azure
+
+
 Usage
 =====
 
diff --git a/sdk/datalake/azure-mgmt-datalake-analytics/setup.py b/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
index 4ba2e4e83639..68f340388091 100644
--- a/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
+++ b/sdk/datalake/azure-mgmt-datalake-analytics/setup.py
@@ -53,7 +53,6 @@
     version=version,
     description='Microsoft Azure {} Client Library for Python'.format(PACKAGE_PPRINT_NAME),
     long_description=readme + '\n\n' + history,
-    long_description_content_type='text/x-rst',
     license='MIT License',
     author='Microsoft Corporation',
     author_email='azpysdkhelp@microsoft.com',
@@ -64,6 +63,7 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',

From d740bb02daae73250d5cf141f374b24f6b0b2f31 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 15:13:16 -0700
Subject: [PATCH 29/47] Trivial code change

---
 .../azure/eventhub/eventprocessor/_ownership_manager.py       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
index e0c3d213c63a..f1c0daf63eb9 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
@@ -83,7 +83,7 @@ async def _balance_ownership(self, all_partition_ids):
         :return: List[Dict[str, Any]], A list of ownership.
         """
         ownership_list = await self.partition_manager.list_ownership(
-            self.eventhub_client.eh_name, self.consumer_group_name
+            self.eventhub_name, self.consumer_group_name
         )
         now = time.time()
         ownership_dict = dict((x["partition_id"], x) for x in ownership_list)  # put the list to dict for fast lookup
@@ -116,7 +116,7 @@ async def _balance_ownership(self, all_partition_ids):
                 random_partition_id = random.choice(claimable_partition_ids)
                 random_chosen_to_claim = ownership_dict.get(random_partition_id,
                                                             {"partition_id": random_partition_id,
-                                                             "eventhub_name": self.eventhub_client.eh_name,
+                                                             "eventhub_name": self.eventhub_name,
                                                              "consumer_group_name": self.consumer_group_name
                                                              })
                 random_chosen_to_claim["owner_id"] = self.owner_id

From 778ab6607e95449e46dd9b332ffbfdce1e4a5295 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Mon, 2 Sep 2019 15:15:29 -0700
Subject: [PATCH 30/47] Add docstring to BlobPartitionManager

---
 .../partitionmanager/blobstorage/blobstoragepm.py      | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
index e28e50873fec..a8b8dcc4de0e 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -14,7 +14,16 @@
 
 
 class BlobPartitionManager(object):
+    """An PartitionManager that uses Azure Blob Storage to store the partition ownership and checkpoint data.
+
+    This class implements the methods (list_ownership, claim_ownership, update_checkpoint) defined in class
+    azure.eventhub.eventprocessor.PartitionManager of package azure-eventhub.
+    """
     def __init__(self, container_client: ContainerClient):
+        """
+
+        :param container_client: The Azure Blob Storage Container client.
+        """
         self._container_client = container_client
         self._cached_ownership_dict = defaultdict(dict)
         self._lock = asyncio.Lock()
@@ -64,6 +73,7 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                     logger.info("Partition %r was claimed by another EventProcessor", partition_id)
                 except Exception as err:
                     logger.warning("Claim error occurred: %r", err)
+                    raise
                 result.append(ownership)
         return result
 

From aad6978f934a3f843adb1de3879a0535c0a3ca8a Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Tue, 3 Sep 2019 03:20:07 -0700
Subject: [PATCH 31/47] Refine exception handling for eventprocessor

---
 .../eventprocessor/event_processor.py         | 69 +++++++++++++------
 .../eventprocessor/partition_manager.py       |  9 ++-
 .../eventprocessor/partition_processor.py     |  1 +
 .../sqlite3_partition_manager.py              |  3 +-
 4 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
index aaeb2de92dd0..e782f9acea52 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
@@ -11,7 +11,7 @@
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
 from .checkpoint_manager import CheckpointManager
-from .partition_manager import PartitionManager
+from .partition_manager import PartitionManager, OwnershipLostError
 from ._ownership_manager import OwnershipManager
 from .partition_processor import CloseReason, PartitionProcessor
 
@@ -182,13 +182,43 @@ async def _receive(self, ownership):
             EventPosition(ownership.get("offset", self._initial_event_position))
         )
 
+        async def process_error(err):
+            log.error(
+                "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                " has met an error. The exception is %r.",
+                owner_id, eventhub_name, partition_id, consumer_group_name, err
+            )
+            try:
+                await partition_processor.process_error(err, checkpoint_manager)
+            except Exception as err_again:  # pylint:disable=broad-except
+                log.error(
+                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                    " has another error during running process_error(). The exception is %r.",
+                    owner_id, eventhub_name, partition_id, consumer_group_name, err_again
+                )
+
+        async def close(reason):
+            log.info(
+                "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                " is being closed. Reason is: %r",
+                owner_id, eventhub_name, partition_id, consumer_group_name, reason
+            )
+            try:
+                await partition_processor.close(reason, checkpoint_manager)
+            except Exception as err:  # pylint:disable=broad-except
+                log.error(
+                    "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
+                    " has an error during running close(). The exception is %r.",
+                    owner_id, eventhub_name, partition_id, consumer_group_name, err
+                )
+
         try:
             while True:
                 try:
                     await partition_processor.initialize()
                     events = await partition_consumer.receive(timeout=self._receive_timeout)
                     await partition_processor.process_events(events, checkpoint_manager)
-                except asyncio.CancelledError as cancelled_error:
+                except asyncio.CancelledError:
                     log.info(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
                         " is cancelled",
@@ -197,28 +227,25 @@ async def _receive(self, ownership):
                         partition_id,
                         consumer_group_name
                     )
-                    await partition_processor.process_error(cancelled_error, checkpoint_manager)
-                    await partition_processor.close(CloseReason.SHUTDOWN, checkpoint_manager)
-                    # TODO: release the ownership immediately via partition manager
+                    if self._running is False:
+                        await close(CloseReason.SHUTDOWN)
+                    else:
+                        await close(CloseReason.OWNERSHIP_LOST)
+                    # TODO: release the ownership immediately via partition manager in preview 4
                     break
                 except EventHubError as eh_err:
-                    reason = CloseReason.OWNERSHIP_LOST if eh_err.error == "link:stolen" \
-                        else CloseReason.EVENTHUB_EXCEPTION
-                    log.warning(
-                        "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
-                        " has met an exception receiving events. It's being closed. The exception is %r.",
-                        owner_id,
-                        eventhub_name,
-                        partition_id,
-                        consumer_group_name,
-                        eh_err
-                    )
-                    await partition_processor.process_error(eh_err, checkpoint_manager)
-                    await partition_processor.close(reason, checkpoint_manager)
+                    await process_error(eh_err)
+                    await close(CloseReason.EVENTHUB_EXCEPTION)
                     # An EventProcessor will pick up this partition again after the ownership is released
-                    # TODO: release the ownership immediately via partition manager
+                    # TODO: release the ownership immediately via partition manager in preview 4
+                    break
+                except OwnershipLostError:
+                    await close(CloseReason.OWNERSHIP_LOST)
+                    break
+                except Exception as other_error:  # pylint:disable=broad-except
+                    await process_error(other_error)
+                    await close(CloseReason.PROCESS_EVENTS_ERROR)
+                    # TODO: release the ownership immediately via partition manager in preview 4
                     break
-                except Exception as exp:  # pylint:disable=broad-except
-                    log.warning(exp)
         finally:
             await partition_consumer.close()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
index e6d3e5df2538..f82ceed383b1 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
@@ -77,8 +77,15 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
         :param sequence_number: The sequence_number of the ~azure.eventhub.EventData the new checkpoint
         will be associated with.
         :type sequence_number: int
-        :return:
+        :return: None
+        :raise: `OwnershipLostError`, `CheckpointError`
         """
 
     async def close(self):
         pass
+
+
+class OwnershipLostError(Exception):
+    """Raises when update_checkpoint detects the ownership has been lost
+
+    """
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
index 37be7006e244..f33983a6fc03 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
@@ -14,6 +14,7 @@ class CloseReason(Enum):
     SHUTDOWN = 0  # user call EventProcessor.stop()
     OWNERSHIP_LOST = 1  # lose the ownership of a partition.
     EVENTHUB_EXCEPTION = 2  # Exception happens during receiving events
+    PROCESS_EVENTS_ERROR = 3  # Exception happens during process_events
 
 
 class PartitionProcessor(ABC):
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
index 50a7d8caf29e..76894184db51 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
@@ -7,7 +7,7 @@
 import uuid
 import sqlite3
 import logging
-from .partition_manager import PartitionManager
+from .partition_manager import PartitionManager, OwnershipLostError
 
 logger = logging.getLogger(__name__)
 
@@ -134,6 +134,7 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             else:
                 logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership",
                             partition_id)
+                raise OwnershipLostError()
 
         finally:
             cursor.close()

From a55dc13765e36f3bff4aca82d51fb5d5b6ead14c Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Tue, 3 Sep 2019 03:20:40 -0700
Subject: [PATCH 32/47] Enable pylint for eventprocessor

---
 pylintrc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pylintrc b/pylintrc
index c5cddb6d9030..399344e8ea99 100644
--- a/pylintrc
+++ b/pylintrc
@@ -2,8 +2,8 @@
 ignore-patterns=test_*,conftest,setup
 reports=no
 
-# PYLINT DIRECTORY BLACKLIST. Ignore eventprocessor temporarily until new eventprocessor code is merged to master
-ignore=_generated,samples,examples,test,tests,doc,.tox,eventprocessor
+# PYLINT DIRECTORY BLACKLIST.
+ignore=_generated,samples,examples,test,tests,doc,.tox
 
 init-hook='import sys; sys.path.insert(0, os.path.abspath(os.getcwd().rsplit("azure-sdk-for-python", 1)[0] + "azure-sdk-for-python/scripts/pylint_custom_plugin"))'
 load-plugins=pylint_guidelines_checker

From a339985bbee3c3793ff399334b5307b6cc1922f0 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Tue, 3 Sep 2019 10:08:52 -0700
Subject: [PATCH 33/47] Expose OwnershipLostError

---
 .../azure-eventhubs/azure/eventhub/eventprocessor/__init__.py  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
index 6280a03fce12..90fa1f2c2e4f 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
@@ -6,7 +6,7 @@
 from .event_processor import EventProcessor
 from .checkpoint_manager import CheckpointManager
 from .partition_processor import PartitionProcessor, CloseReason
-from .partition_manager import PartitionManager
+from .partition_manager import PartitionManager, OwnershipLostError
 from .sqlite3_partition_manager import Sqlite3PartitionManager
 
 __all__ = [
@@ -16,4 +16,5 @@
     'PartitionManager',
     'CheckpointManager',
     'Sqlite3PartitionManager',
+    'OwnershipLostError',
 ]

From 9bed566d26d0645fb10a933bf7643a0e116b71af Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Tue, 3 Sep 2019 10:11:35 -0700
Subject: [PATCH 34/47] Refine exception handling

---
 .../blobstorage/blobstoragepm.py              | 55 +++++++++++++------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
index a8b8dcc4de0e..20af59ff03cf 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -6,14 +6,15 @@
 import logging
 from collections import defaultdict
 import asyncio
-from azure.core.exceptions import ResourceModifiedError, ResourceExistsError
+from azure.eventhub.eventprocessor import PartitionManager, OwnershipLostError
+from azure.core.exceptions import ResourceModifiedError, ResourceExistsError, AzureError
 from azure.storage.blob.aio import ContainerClient
 
 logger = logging.getLogger(__name__)
 UPLOAD_DATA = ""
 
 
-class BlobPartitionManager(object):
+class BlobPartitionManager(PartitionManager):
     """An PartitionManager that uses Azure Blob Storage to store the partition ownership and checkpoint data.
 
     This class implements the methods (list_ownership, claim_ownership, update_checkpoint) defined in class
@@ -25,13 +26,18 @@ def __init__(self, container_client: ContainerClient):
         :param container_client: The Azure Blob Storage Container client.
         """
         self._container_client = container_client
-        self._cached_ownership_dict = defaultdict(dict)
+        self._cached_ownership_dict = defaultdict(dict)  # type: Dict[str, Dict[str, Any]]
         self._lock = asyncio.Lock()
 
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
         async with self._lock:
-            blobs = self._container_client.list_blobs(include=['metadata'])
-            # result = []
+            try:
+                blobs = self._container_client.list_blobs(include=['metadata'])
+            except AzureError as azure_err:  # list_blobs has exhausted retry
+                logger.exception("An exception occurred during list_ownership for eventhub %r consumer group %r."
+                                 " An empty ownership list is returned",
+                                 eventhub_name, consumer_group_name, exc_info=azure_err)
+                return []
             async for b in blobs:
                 metadata = b.metadata
                 ownership = {
@@ -56,12 +62,16 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                 if "sequence_number" in ownership:
                     metadata["sequence_number"] = ownership["sequence_number"]
                 partition_id = ownership["partition_id"]
+                eventhub_name = ownership["eventhub_name"]
+                consumer_group_name = ownership["consumer_group_name"]
+                owner_id = ownership["owner_id"]
+
+                etag = ownership.get("etag")
+                if etag:
+                    etag_match = {"if_match": '"'+etag+'"'}
+                else:
+                    etag_match = {"if_none_match": '"*"'}
                 try:
-                    etag = ownership.get("etag")
-                    if etag:
-                        etag_match = {"if_match": '"'+etag+'"'}
-                    else:
-                        etag_match = {"if_none_match": '"*"'}
                     blob_client = await self._container_client.upload_blob(
                         name=partition_id, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
                     )
@@ -70,11 +80,15 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                     ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
                     self._cached_ownership_dict[partition_id] = ownership
                 except (ResourceModifiedError, ResourceExistsError):
-                    logger.info("Partition %r was claimed by another EventProcessor", partition_id)
+                    logger.info(
+                        "EventProcessor instance %r of eventhub %r consumer group %r lost ownership to partition %r",
+                        owner_id, eventhub_name, consumer_group_name, partition_id)
                 except Exception as err:
-                    logger.warning("Claim error occurred: %r", err)
-                    raise
-                result.append(ownership)
+                    logger.exception("An exception occurred when EventProcessor instance %r claim_ownership for "
+                                     "eventhub %r consumer group %r partition %r. The ownership is now lost",
+                                     owner_id, eventhub_name, consumer_group_name, partition_id, exc_info=err)
+                else:
+                    result.append(ownership)
         return result
 
     async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
@@ -94,7 +108,14 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
                 cached_ownership["etag"] = uploaded_blob_properties.etag
                 cached_ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
             except (ResourceModifiedError, ResourceExistsError):
-                logger.info("Partition %r was claimed by another EventProcessor", partition_id)
+                logger.info(
+                    "EventProcessor instance %r of eventhub %r consumer group %r couldn't update_checkpoint to "
+                    "partition %r because the ownership has been stolen",
+                    owner_id, eventhub_name, consumer_group_name, partition_id)
+                raise OwnershipLostError()
             except Exception as err:
-                logger.warning("Checkpoint error occurred: %r", err)
-                raise
+                logger.exception(
+                    "EventProcessor instance %r of eventhub %r consumer group %r couldn't update_checkpoint to "
+                    "partition %r because of unexpected error",
+                    owner_id, eventhub_name, consumer_group_name, partition_id, exc_info=err)
+                raise  # EventProcessor will catch the exception and handle it

From cbc6792458607b22cd8f697d61c867250ad3cba0 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Tue, 3 Sep 2019 23:53:58 -0700
Subject: [PATCH 35/47] handle exception for claim_ownership and
 update_checkpoint

---
 .../partitionmanager/blobstorage/blobstoragepm.py        | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
index 20af59ff03cf..90680d225def 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
@@ -17,8 +17,9 @@
 class BlobPartitionManager(PartitionManager):
     """An PartitionManager that uses Azure Blob Storage to store the partition ownership and checkpoint data.
 
-    This class implements the methods (list_ownership, claim_ownership, update_checkpoint) defined in class
+    This class implements methods list_ownership, claim_ownership, and update_checkpoint that are defined in class
     azure.eventhub.eventprocessor.PartitionManager of package azure-eventhub.
+
     """
     def __init__(self, container_client: ContainerClient):
         """
@@ -27,7 +28,7 @@ def __init__(self, container_client: ContainerClient):
         """
         self._container_client = container_client
         self._cached_ownership_dict = defaultdict(dict)  # type: Dict[str, Dict[str, Any]]
-        self._lock = asyncio.Lock()
+        self._lock = asyncio.Lock()  # TODO: consider locking every ownership/partition instead of whole thing.
 
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
         async with self._lock:
@@ -38,7 +39,7 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
                                  " An empty ownership list is returned",
                                  eventhub_name, consumer_group_name, exc_info=azure_err)
                 return []
-            async for b in blobs:
+            async for b in blobs:  # TODO: consider running them concurrently
                 metadata = b.metadata
                 ownership = {
                     "eventhub_name": eventhub_name,
@@ -55,7 +56,7 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
     async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
         result = []
         async with self._lock:
-            for ownership in ownership_list:
+            for ownership in ownership_list:  # TODO: consider claiming concurrently
                 metadata = {"owner_id": ownership["owner_id"]}
                 if "offset" in ownership:
                     metadata["offset"] = ownership["offset"]

From 8748e1f4eb392b8243e5220f4215e38c38aa94d6 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 00:01:45 -0700
Subject: [PATCH 36/47] Add license info

---
 .../azure/__init__.py                                        | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
index e69de29bb2d1..62351a0ab30b 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
+++ b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

From 97e05583e4940379da8845b443151dfc87a10941 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 00:55:28 -0700
Subject: [PATCH 37/47] Restructure packages

---
 .../azure/ehplugin/__init__.py                       |  0
 .../azure/ehplugin/partitionmanager/__init__.py      |  0
 .../HISTORY.md                                       |  0
 .../LICENSE                                          |  0
 .../MANIFEST.in                                      |  0
 .../README.md                                        |  0
 .../azure/__init__.py                                |  0
 .../azure/eventhub}/__init__.py                      |  3 +--
 .../azure/eventhub/aio/__init__.py                   |  5 +++++
 .../azure/eventhub/aio/partitionmanager/__init__.py  |  5 +++++
 .../aio/partitionmanager/blobstorage/__init__.py     | 12 ++++++++++++
 .../partitionmanager/blobstorage/blobstoragepm.py    |  0
 .../conftest.py                                      |  0
 .../dev_requirements.txt                             |  0
 .../examples/event_processor_blob_storage_example.py |  2 +-
 .../sdk_packaging.toml                               |  0
 .../setup.cfg                                        |  0
 .../setup.py                                         |  2 +-
 18 files changed, 25 insertions(+), 4 deletions(-)
 delete mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
 delete mode 100644 sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/HISTORY.md (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/LICENSE (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/MANIFEST.in (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/README.md (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/azure/__init__.py (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub}/__init__.py (80%)
 create mode 100644 sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py
 create mode 100644 sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py
 create mode 100644 sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin => azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio}/partitionmanager/blobstorage/blobstoragepm.py (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/conftest.py (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/dev_requirements.txt (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/examples/event_processor_blob_storage_example.py (95%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/sdk_packaging.toml (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/setup.cfg (100%)
 rename sdk/eventhub/{azure-ehplugin-partitionmanager-blobstorage => azure-eventhubs-aio-partitionmanager-blobstorage}/setup.py (97%)

diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/__init__.py
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py b/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/__init__.py
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/HISTORY.md
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/HISTORY.md
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/HISTORY.md
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/LICENSE
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/LICENSE
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/LICENSE
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/MANIFEST.in
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/MANIFEST.in
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/MANIFEST.in
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/README.md
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/README.md
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/README.md
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/__init__.py
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/__init__.py
similarity index 80%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/__init__.py
index 3057ad223050..62351a0ab30b 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/__init__.py
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/__init__.py
@@ -2,5 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
-
-__version__ = "1.0.0b1"
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py
new file mode 100644
index 000000000000..62351a0ab30b
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py
new file mode 100644
index 000000000000..62351a0ab30b
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py
new file mode 100644
index 000000000000..9fce432c2636
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py
@@ -0,0 +1,12 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+__version__ = "1.0.0b1"
+
+from .blobstoragepm import BlobPartitionManager
+
+__all__ = [
+    BlobPartitionManager
+]
\ No newline at end of file
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/azure/ehplugin/partitionmanager/blobstorage/blobstoragepm.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/conftest.py
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/conftest.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/conftest.py
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/dev_requirements.txt
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/dev_requirements.txt
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/dev_requirements.txt
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
similarity index 95%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
index 48c30689a4ea..46830b372965 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
@@ -3,7 +3,7 @@
 import os
 from azure.eventhub.aio import EventHubClient
 from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
-from azure.ehplugin.partitionmanager.blobstorage.blobstoragepm import BlobPartitionManager
+from azure.eventhub.aio.partitionmanager.blobstorage import BlobPartitionManager
 from azure.storage.blob.aio import ContainerClient
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/sdk_packaging.toml
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/sdk_packaging.toml
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/sdk_packaging.toml
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.cfg
similarity index 100%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.cfg
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.cfg
diff --git a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py
similarity index 97%
rename from sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
rename to sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py
index 591569ab023c..98630edbdcc4 100644
--- a/sdk/eventhub/azure-ehplugin-partitionmanager-blobstorage/setup.py
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py
@@ -14,7 +14,7 @@
 
 
 # Change the PACKAGE_NAME only to change folder and different name
-PACKAGE_NAME = "azure-ehplugin-partitionmanager-blobstorage"
+PACKAGE_NAME = "azure-eventhub-aio-partitionmanager-blobstorage"
 PACKAGE_PPRINT_NAME = "Event Hubs Event Procesor Partition Manager implementation with Blob Storage"
 
 # a-b-c => a/b/c

From e61d6a1ebc833ec9ae33ec5ace92cef0e8a6db2d Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 10:03:00 -0700
Subject: [PATCH 38/47] Lock each ownership with a separate lock

---
 .../blobstorage/blobstoragepm.py              | 37 ++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
index 90680d225def..be7c1a1ef1d0 100644
--- a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
@@ -28,18 +28,20 @@ def __init__(self, container_client: ContainerClient):
         """
         self._container_client = container_client
         self._cached_ownership_dict = defaultdict(dict)  # type: Dict[str, Dict[str, Any]]
-        self._lock = asyncio.Lock()  # TODO: consider locking every ownership/partition instead of whole thing.
+        # lock each partition for list_ownership, claim_ownership and update_checkpoint etag doesn't get out of sync
+        # when the three methods are running concurrently
+        self._cached_ownership_locks = defaultdict(asyncio.Lock)
 
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
-        async with self._lock:
-            try:
-                blobs = self._container_client.list_blobs(include=['metadata'])
-            except AzureError as azure_err:  # list_blobs has exhausted retry
-                logger.exception("An exception occurred during list_ownership for eventhub %r consumer group %r."
-                                 " An empty ownership list is returned",
-                                 eventhub_name, consumer_group_name, exc_info=azure_err)
-                return []
-            async for b in blobs:  # TODO: consider running them concurrently
+        try:
+            blobs = self._container_client.list_blobs(include=['metadata'])
+        except AzureError as azure_err:  # list_blobs has exhausted retry
+            logger.exception("An exception occurred during list_ownership for eventhub %r consumer group %r."
+                             " An empty ownership list is returned",
+                             eventhub_name, consumer_group_name, exc_info=azure_err)
+            return []
+        async for b in blobs:  # TODO: running them concurrently
+            async with self._cached_ownership_locks[b.name]:
                 metadata = b.metadata
                 ownership = {
                     "eventhub_name": eventhub_name,
@@ -55,17 +57,18 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
 
     async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
         result = []
-        async with self._lock:
-            for ownership in ownership_list:  # TODO: consider claiming concurrently
+        for ownership in ownership_list:  # TODO: claiming concurrently
+            partition_id = ownership["partition_id"]
+            eventhub_name = ownership["eventhub_name"]
+            consumer_group_name = ownership["consumer_group_name"]
+            owner_id = ownership["owner_id"]
+
+            async with self._cached_ownership_locks[partition_id]:
                 metadata = {"owner_id": ownership["owner_id"]}
                 if "offset" in ownership:
                     metadata["offset"] = ownership["offset"]
                 if "sequence_number" in ownership:
                     metadata["sequence_number"] = ownership["sequence_number"]
-                partition_id = ownership["partition_id"]
-                eventhub_name = ownership["eventhub_name"]
-                consumer_group_name = ownership["consumer_group_name"]
-                owner_id = ownership["owner_id"]
 
                 etag = ownership.get("etag")
                 if etag:
@@ -100,7 +103,7 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             "offset": offset,
             "sequence_number": str(sequence_number)
         }
-        async with self._lock:
+        async with self._cached_ownership_locks[partition_id]:
             try:
                 blob_client = await self._container_client.upload_blob(
                     name=partition_id, data=UPLOAD_DATA, metadata=metadata, overwrite=True)

From 9102713c18db2f5c5799a4d0ef9cfa2d0483b005 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 12:12:08 -0700
Subject: [PATCH 39/47] Move eventprocessor to aio rename
 Sqlite3PartitionManager to SamplePartitionManager

---
 .../eventhub/{ => aio}/eventprocessor/__init__.py      |  3 ---
 .../{ => aio}/eventprocessor/_ownership_manager.py     |  0
 .../{ => aio}/eventprocessor/checkpoint_manager.py     |  0
 .../{ => aio}/eventprocessor/event_processor.py        |  0
 .../{ => aio}/eventprocessor/partition_manager.py      |  0
 .../{ => aio}/eventprocessor/partition_processor.py    |  4 ++--
 .../aio/eventprocessor/partitionmanagers/__init__.py   | 10 ++++++++++
 .../partitionmanagers/sample_partition_manager.py}     |  7 ++++---
 .../azure/eventhub/{ => aio}/eventprocessor/utils.py   |  0
 .../examples/eventprocessor/event_processor_example.py |  6 +++---
 10 files changed, 19 insertions(+), 11 deletions(-)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/__init__.py (85%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/_ownership_manager.py (100%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/checkpoint_manager.py (100%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/event_processor.py (100%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/partition_manager.py (100%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/partition_processor.py (97%)
 create mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{eventprocessor/sqlite3_partition_manager.py => aio/eventprocessor/partitionmanagers/sample_partition_manager.py} (96%)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/{ => aio}/eventprocessor/utils.py (100%)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
similarity index 85%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
index 90fa1f2c2e4f..86810f6053b4 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/__init__.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
@@ -7,14 +7,11 @@
 from .checkpoint_manager import CheckpointManager
 from .partition_processor import PartitionProcessor, CloseReason
 from .partition_manager import PartitionManager, OwnershipLostError
-from .sqlite3_partition_manager import Sqlite3PartitionManager
 
 __all__ = [
     'CloseReason',
     'EventProcessor',
     'PartitionProcessor',
     'PartitionManager',
-    'CheckpointManager',
-    'Sqlite3PartitionManager',
     'OwnershipLostError',
 ]
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/_ownership_manager.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/_ownership_manager.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/_ownership_manager.py
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/checkpoint_manager.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/checkpoint_manager.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/checkpoint_manager.py
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/event_processor.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_manager.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_manager.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_manager.py
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
similarity index 97%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
index f33983a6fc03..2f46145bfb76 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
@@ -4,7 +4,7 @@
 # -----------------------------------------------------------------------------------
 
 from typing import List
-from abc import ABC
+from abc import ABC, abstractmethod
 from enum import Enum
 from azure.eventhub import EventData
 from .checkpoint_manager import CheckpointManager
@@ -41,6 +41,7 @@ async def close(self, reason, checkpoint_manager: CheckpointManager):
 
         """
 
+    @abstractmethod
     async def process_events(self, events: List[EventData], checkpoint_manager: CheckpointManager):
         """Called when a batch of events have been received.
 
@@ -50,7 +51,6 @@ async def process_events(self, events: List[EventData], checkpoint_manager: Chec
         :type checkpoint_manager: ~azure.eventhub.CheckpointManager
 
         """
-        raise NotImplementedError
 
     async def process_error(self, error, checkpoint_manager: CheckpointManager):
         """Called when an error happens
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
new file mode 100644
index 000000000000..d81fee2a97cd
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
@@ -0,0 +1,10 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# -----------------------------------------------------------------------------------
+
+from .sample_partition_manager import SamplePartitionManager
+
+__all__ = [
+    'SamplePartitionManager',
+]
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py
similarity index 96%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py
index 76894184db51..4859d675d5b9 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/sqlite3_partition_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py
@@ -7,7 +7,7 @@
 import uuid
 import sqlite3
 import logging
-from .partition_manager import PartitionManager, OwnershipLostError
+from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError
 
 logger = logging.getLogger(__name__)
 
@@ -19,9 +19,10 @@ def _check_table_name(table_name: str):
     return table_name
 
 
-class Sqlite3PartitionManager(PartitionManager):
+class SamplePartitionManager(PartitionManager):
     """An implementation of PartitionManager by using the sqlite3 in Python standard library.
     Sqlite3 is a mini sql database that runs in memory or files.
+    Please don't use this PartitionManager for production use.
 
 
     """
@@ -41,7 +42,7 @@ def __init__(self, db_filename: str = ":memory:", ownership_table: str = "owners
         Sqlite3 will run in memory without a file when db_filename is ":memory:".
         :param ownership_table: The table name of the sqlite3 database.
         """
-        super(Sqlite3PartitionManager, self).__init__()
+        super(SamplePartitionManager, self).__init__()
         self.ownership_table = _check_table_name(ownership_table)
         conn = sqlite3.connect(db_filename)
         c = conn.cursor()
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/utils.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/eventprocessor/utils.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/utils.py
diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
index c17bb95bd4cf..de1b88fc4203 100644
--- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
+++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
@@ -2,8 +2,8 @@
 import logging
 import os
 from azure.eventhub.aio import EventHubClient
-from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
-from azure.eventhub.eventprocessor import Sqlite3PartitionManager
+from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor
+from azure.eventhub.aio.eventprocessor.partitionmanagers import SamplePartitionManager
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
 RETRY_TOTAL = 3  # max number of retries for receive operations within the receive timeout. Actual number of retries clould be less if RECEIVE_TIMEOUT is too small
@@ -29,7 +29,7 @@ async def process_events(self, events, checkpoint_manager):
 if __name__ == '__main__':
     loop = asyncio.get_event_loop()
     client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
-    partition_manager = Sqlite3PartitionManager(db_filename="eventprocessor_test_db")
+    partition_manager = SamplePartitionManager(db_filename="eventprocessor_test_db")
     event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=1)
     try:
         loop.run_until_complete(event_processor.start())

From 278592cfdb6d549515ed2b9e869713af4152d52b Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 13:48:06 -0700
Subject: [PATCH 40/47] change checkpoint_manager to partition context

---
 .../eventhub/aio/eventprocessor/__init__.py   |  3 +-
 .../aio/eventprocessor/event_processor.py     | 16 +++++-----
 ...kpoint_manager.py => partition_context.py} | 12 ++++----
 .../aio/eventprocessor/partition_processor.py | 30 ++++++++++++-------
 .../eventprocessor/event_processor_example.py |  6 ++--
 5 files changed, 38 insertions(+), 29 deletions(-)
 rename sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/{checkpoint_manager.py => partition_context.py} (78%)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
index 86810f6053b4..c7cd2def61a7 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/__init__.py
@@ -4,9 +4,9 @@
 # -----------------------------------------------------------------------------------
 
 from .event_processor import EventProcessor
-from .checkpoint_manager import CheckpointManager
 from .partition_processor import PartitionProcessor, CloseReason
 from .partition_manager import PartitionManager, OwnershipLostError
+from .partition_context import PartitionContext
 
 __all__ = [
     'CloseReason',
@@ -14,4 +14,5 @@
     'PartitionProcessor',
     'PartitionManager',
     'OwnershipLostError',
+    'PartitionContext',
 ]
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
index e782f9acea52..b028d960e36b 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
@@ -10,7 +10,7 @@
 
 from azure.eventhub import EventPosition, EventHubError
 from azure.eventhub.aio import EventHubClient
-from .checkpoint_manager import CheckpointManager
+from .partition_context import PartitionContext
 from .partition_manager import PartitionManager, OwnershipLostError
 from ._ownership_manager import OwnershipManager
 from .partition_processor import CloseReason, PartitionProcessor
@@ -36,10 +36,10 @@ class EventProcessor(object):  # pylint:disable=too-many-instance-attributes
             from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor, Sqlite3PartitionManager
 
             class MyPartitionProcessor(object):
-                async def process_events(self, events, checkpoint_manager):
+                async def process_events(self, events, partition_context):
                     if events:
                         # do something sync or async to process the events
-                        await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+                        await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number)
 
 
             client = EventHubClient.from_connection_string("<your connection string>", receive_timeout=5, retry_total=3)
@@ -167,7 +167,7 @@ async def _receive(self, ownership):
         eventhub_name = ownership["eventhub_name"]
         consumer_group_name = ownership["consumer_group_name"]
         owner_id = ownership["owner_id"]
-        checkpoint_manager = CheckpointManager(
+        partition_context = PartitionContext(
             eventhub_name,
             consumer_group_name,
             partition_id,
@@ -175,7 +175,7 @@ async def _receive(self, ownership):
             self._partition_manager
         )
         partition_processor.eventhub_name = ownership
-        partition_processor.checkpoint_manager = checkpoint_manager
+        partition_processor._partition_context = partition_context
         partition_consumer = self._eventhub_client.create_consumer(
             consumer_group_name,
             partition_id,
@@ -189,7 +189,7 @@ async def process_error(err):
                 owner_id, eventhub_name, partition_id, consumer_group_name, err
             )
             try:
-                await partition_processor.process_error(err, checkpoint_manager)
+                await partition_processor.process_error(err, partition_context)
             except Exception as err_again:  # pylint:disable=broad-except
                 log.error(
                     "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
@@ -204,7 +204,7 @@ async def close(reason):
                 owner_id, eventhub_name, partition_id, consumer_group_name, reason
             )
             try:
-                await partition_processor.close(reason, checkpoint_manager)
+                await partition_processor.close(reason, partition_context)
             except Exception as err:  # pylint:disable=broad-except
                 log.error(
                     "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
@@ -217,7 +217,7 @@ async def close(reason):
                 try:
                     await partition_processor.initialize()
                     events = await partition_consumer.receive(timeout=self._receive_timeout)
-                    await partition_processor.process_events(events, checkpoint_manager)
+                    await partition_processor.process_events(events, partition_context)
                 except asyncio.CancelledError:
                     log.info(
                         "PartitionProcessor of EventProcessor instance %r of eventhub %r partition %r consumer group %r"
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/checkpoint_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py
similarity index 78%
rename from sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/checkpoint_manager.py
rename to sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py
index 3be0c5d945c3..6aaf939143a2 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/checkpoint_manager.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_context.py
@@ -7,11 +7,10 @@
 from .partition_manager import PartitionManager
 
 
-class CheckpointManager(object):
-    """
-    CheckpointManager is responsible for the creation of checkpoints.
-    The interaction with the chosen storage service is done via ~azure.eventhub.eventprocessor.PartitionManager.
+class PartitionContext(object):
+    """Contains partition related context information for a PartitionProcessor instance to use.
 
+    Users can use update_checkpoint() of this class to save checkpoint data.
     """
     def __init__(self, eventhub_name: str, consumer_group_name: str,
                  partition_id: str, owner_id: str, partition_manager: PartitionManager):
@@ -19,7 +18,7 @@ def __init__(self, eventhub_name: str, consumer_group_name: str,
         self.eventhub_name = eventhub_name
         self.consumer_group_name = consumer_group_name
         self.owner_id = owner_id
-        self.partition_manager = partition_manager
+        self._partition_manager = partition_manager
 
     async def update_checkpoint(self, offset, sequence_number=None):
         """
@@ -33,7 +32,8 @@ async def update_checkpoint(self, offset, sequence_number=None):
         :type sequence_number: int
         :return: None
         """
-        await self.partition_manager.update_checkpoint(
+        # TODO: whether change this method to accept event_data as well
+        await self._partition_manager.update_checkpoint(
             self.eventhub_name, self.consumer_group_name, self.partition_id, self.owner_id, offset,
             sequence_number
         )
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
index 2f46145bfb76..ecbbe75d543d 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
@@ -7,7 +7,7 @@
 from abc import ABC, abstractmethod
 from enum import Enum
 from azure.eventhub import EventData
-from .checkpoint_manager import CheckpointManager
+from .partition_context import PartitionContext
 
 
 class CloseReason(Enum):
@@ -25,10 +25,15 @@ class PartitionProcessor(ABC):
 
     """
 
-    async def initialize(self):
+    async def initialize(self, partition_context: PartitionContext):
+        """
+
+        :param partition_context: The context information of this partition.
+        :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext
+        """
         pass
 
-    async def close(self, reason, checkpoint_manager: CheckpointManager):
+    async def close(self, reason, partition_context: PartitionContext):
         """Called when EventProcessor stops processing this PartitionProcessor.
 
         There are different reasons to trigger the PartitionProcessor to close.
@@ -36,28 +41,31 @@ async def close(self, reason, checkpoint_manager: CheckpointManager):
 
         :param reason: Reason for closing the PartitionProcessor.
         :type reason: ~azure.eventhub.eventprocessor.CloseReason
-        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
-        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
+        :param partition_context: The context information of this partition.
+        Use its method update_checkpoint to save checkpoint to the data store.
+        :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext
 
         """
 
     @abstractmethod
-    async def process_events(self, events: List[EventData], checkpoint_manager: CheckpointManager):
+    async def process_events(self, events: List[EventData], partition_context: PartitionContext):
         """Called when a batch of events have been received.
 
         :param events: Received events.
         :type events: list[~azure.eventhub.common.EventData]
-        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
-        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
+        :param partition_context: The context information of this partition.
+        Use its method update_checkpoint to save checkpoint to the data store.
+        :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext
 
         """
 
-    async def process_error(self, error, checkpoint_manager: CheckpointManager):
+    async def process_error(self, error, partition_context: PartitionContext):
         """Called when an error happens
 
         :param error: The error that happens.
         :type error: Exception
-        :param checkpoint_manager: Use its method update_checkpoint to update checkpoint to the data store
-        :type checkpoint_manager: ~azure.eventhub.CheckpointManager
+        :param partition_context: The context information of this partition.
+        Use its method update_checkpoint to save checkpoint to the data store.
+        :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext
 
         """
diff --git a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
index de1b88fc4203..c991a88d20de 100644
--- a/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
+++ b/sdk/eventhub/azure-eventhubs/examples/eventprocessor/event_processor_example.py
@@ -18,12 +18,12 @@ async def do_operation(event):
 
 
 class MyPartitionProcessor(PartitionProcessor):
-    async def process_events(self, events, checkpoint_manager):
+    async def process_events(self, events, partition_context):
         if events:
             await asyncio.gather(*[do_operation(event) for event in events])
-            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+            await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number)
         else:
-            print("empty events received", "partition:", checkpoint_manager.partition_id)
+            print("empty events received", "partition:", partition_context.partition_id)
 
 
 if __name__ == '__main__':

From 665f28cd7251945f1428b397017ec7c239548fa7 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 13:51:23 -0700
Subject: [PATCH 41/47] fix pylint error

---
 .../azure/eventhub/aio/eventprocessor/event_processor.py         | 1 -
 .../azure/eventhub/aio/eventprocessor/partition_processor.py     | 1 -
 2 files changed, 2 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
index b028d960e36b..0ce70990f85f 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/event_processor.py
@@ -175,7 +175,6 @@ async def _receive(self, ownership):
             self._partition_manager
         )
         partition_processor.eventhub_name = ownership
-        partition_processor._partition_context = partition_context
         partition_consumer = self._eventhub_client.create_consumer(
             consumer_group_name,
             partition_id,
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
index ecbbe75d543d..8b0fb2ca7e5c 100644
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
+++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partition_processor.py
@@ -31,7 +31,6 @@ async def initialize(self, partition_context: PartitionContext):
         :param partition_context: The context information of this partition.
         :type partition_context: ~azure.eventhub.aio.eventprocessor.PartitionContext
         """
-        pass
 
     async def close(self, reason, partition_context: PartitionContext):
         """Called when EventProcessor stops processing this PartitionProcessor.

From db93fd4196ef868d23f9221bd2f9c5d166c84df3 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 15:35:46 -0700
Subject: [PATCH 42/47] Re-org namespace package structure

---
 .../HISTORY.md                                               | 0
 .../LICENSE                                                  | 0
 .../MANIFEST.in                                              | 0
 .../README.md                                                | 0
 .../azure/__init__.py                                        | 0
 .../azure/eventhub/__init__.py                               | 0
 .../azure/eventhub/aio/__init__.py                           | 0
 .../azure/eventhub/aio/eventprocessor}/__init__.py           | 0
 .../aio/eventprocessor/partitionmanagers/__init__.py         | 5 +++++
 .../partitionmanagers}/blobstorage/__init__.py               | 4 ++--
 .../partitionmanagers}/blobstorage/blobstoragepm.py          | 2 +-
 .../conftest.py                                              | 0
 .../dev_requirements.txt                                     | 0
 .../examples/event_processor_blob_storage_example.py         | 4 ++--
 .../sdk_packaging.toml                                       | 0
 .../setup.cfg                                                | 0
 .../setup.py                                                 | 2 +-
 17 files changed, 11 insertions(+), 6 deletions(-)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/HISTORY.md (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/LICENSE (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/MANIFEST.in (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/README.md (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/azure/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/azure/eventhub/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/azure/eventhub/aio/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor}/__init__.py (100%)
 create mode 100644 sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers}/blobstorage/__init__.py (93%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers}/blobstorage/blobstoragepm.py (98%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/conftest.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/dev_requirements.txt (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/examples/event_processor_blob_storage_example.py (91%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/sdk_packaging.toml (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/setup.cfg (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-partitionmanager-blobstorage => azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage}/setup.py (96%)

diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/HISTORY.md b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/HISTORY.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/HISTORY.md
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/HISTORY.md
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/LICENSE b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/LICENSE
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/LICENSE
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/LICENSE
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/MANIFEST.in b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/MANIFEST.in
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/MANIFEST.in
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/MANIFEST.in
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/README.md b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/README.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/README.md
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/README.md
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
new file mode 100644
index 000000000000..62351a0ab30b
--- /dev/null
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
@@ -0,0 +1,5 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/__init__.py
similarity index 93%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/__init__.py
index 9fce432c2636..9e0e473c9a8b 100644
--- a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/__init__.py
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/__init__.py
@@ -8,5 +8,5 @@
 from .blobstoragepm import BlobPartitionManager
 
 __all__ = [
-    BlobPartitionManager
-]
\ No newline at end of file
+    "BlobPartitionManager",
+]
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
similarity index 98%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
index be7c1a1ef1d0..bfefaf7f320e 100644
--- a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/azure/eventhub/aio/partitionmanager/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
@@ -6,7 +6,7 @@
 import logging
 from collections import defaultdict
 import asyncio
-from azure.eventhub.eventprocessor import PartitionManager, OwnershipLostError
+from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError
 from azure.core.exceptions import ResourceModifiedError, ResourceExistsError, AzureError
 from azure.storage.blob.aio import ContainerClient
 
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/conftest.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/conftest.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/conftest.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/conftest.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/dev_requirements.txt b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/dev_requirements.txt
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/dev_requirements.txt
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/dev_requirements.txt
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py
similarity index 91%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py
index 46830b372965..e371971f7029 100644
--- a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py
@@ -2,8 +2,8 @@
 import logging
 import os
 from azure.eventhub.aio import EventHubClient
-from azure.eventhub.eventprocessor import EventProcessor, PartitionProcessor
-from azure.eventhub.aio.partitionmanager.blobstorage import BlobPartitionManager
+from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor
+from azure.eventhub.aio.eventprocessor.partitionmanagers.blobstorage import BlobPartitionManager
 from azure.storage.blob.aio import ContainerClient
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/sdk_packaging.toml
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/sdk_packaging.toml
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/sdk_packaging.toml
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.cfg b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.cfg
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.cfg
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.cfg
diff --git a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py
similarity index 96%
rename from sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py
rename to sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py
index 98630edbdcc4..8f7ce31a18f3 100644
--- a/sdk/eventhub/azure-eventhubs-aio-partitionmanager-blobstorage/setup.py
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py
@@ -14,7 +14,7 @@
 
 
 # Change the PACKAGE_NAME only to change folder and different name
-PACKAGE_NAME = "azure-eventhub-aio-partitionmanager-blobstorage"
+PACKAGE_NAME = "azure-eventhub-aio-eventprocessor-partitionmanagers-blobstorage"
 PACKAGE_PPRINT_NAME = "Event Hubs Event Procesor Partition Manager implementation with Blob Storage"
 
 # a-b-c => a/b/c

From 2050615e9900ba78efa37b2d5d1a362dff49ce88 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Wed, 4 Sep 2019 20:23:44 -0700
Subject: [PATCH 43/47] raise error while list_ownership got an exception

---
 .../blobstorage/blobstoragepm.py              | 27 +++++++++----------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
index bfefaf7f320e..1de8b75ac2f7 100644
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
@@ -35,11 +35,10 @@ def __init__(self, container_client: ContainerClient):
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
         try:
             blobs = self._container_client.list_blobs(include=['metadata'])
-        except AzureError as azure_err:  # list_blobs has exhausted retry
-            logger.exception("An exception occurred during list_ownership for eventhub %r consumer group %r."
-                             " An empty ownership list is returned",
-                             eventhub_name, consumer_group_name, exc_info=azure_err)
-            return []
+        except Exception as err:  # pylint:disable=broad-except
+            logger.warning("An exception occurred during list_ownership for eventhub %r consumer group %r. "
+                           "Exception is %r", eventhub_name, consumer_group_name, err)
+            raise
         async for b in blobs:  # TODO: running them concurrently
             async with self._cached_ownership_locks[b.name]:
                 metadata = b.metadata
@@ -74,7 +73,7 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                 if etag:
                     etag_match = {"if_match": '"'+etag+'"'}
                 else:
-                    etag_match = {"if_none_match": '"*"'}
+                    etag_match = {"if_none_match": '*'}
                 try:
                     blob_client = await self._container_client.upload_blob(
                         name=partition_id, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
@@ -83,16 +82,16 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                     ownership["etag"] = uploaded_blob_properties.etag
                     ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
                     self._cached_ownership_dict[partition_id] = ownership
+                    result.append(ownership)
                 except (ResourceModifiedError, ResourceExistsError):
                     logger.info(
                         "EventProcessor instance %r of eventhub %r consumer group %r lost ownership to partition %r",
                         owner_id, eventhub_name, consumer_group_name, partition_id)
                 except Exception as err:
-                    logger.exception("An exception occurred when EventProcessor instance %r claim_ownership for "
-                                     "eventhub %r consumer group %r partition %r. The ownership is now lost",
-                                     owner_id, eventhub_name, consumer_group_name, partition_id, exc_info=err)
-                else:
-                    result.append(ownership)
+                    logger.warning("An exception occurred when EventProcessor instance %r claim_ownership for "
+                                   "eventhub %r consumer group %r partition %r. The ownership is now lost. Exception "
+                                   "is %r", owner_id, eventhub_name, consumer_group_name, partition_id, err)
+
         return result
 
     async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
@@ -118,8 +117,8 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
                     owner_id, eventhub_name, consumer_group_name, partition_id)
                 raise OwnershipLostError()
             except Exception as err:
-                logger.exception(
+                logger.warning(
                     "EventProcessor instance %r of eventhub %r consumer group %r couldn't update_checkpoint to "
-                    "partition %r because of unexpected error",
-                    owner_id, eventhub_name, consumer_group_name, partition_id, exc_info=err)
+                    "partition %r because of unexpected error. Exception is",
+                    owner_id, eventhub_name, consumer_group_name, partition_id, err)
                 raise  # EventProcessor will catch the exception and handle it

From 2781062fb6578e55d2b2c92ea8bf11f57b97f981 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Thu, 5 Sep 2019 15:48:56 -0700
Subject: [PATCH 44/47] Restructure package structure

---
 .../eventhub/aio/eventprocessor/__init__.py   |  5 --
 .../partitionmanagers/__init__.py             |  5 --
 .../HISTORY.md                                |  0
 .../LICENSE                                   |  0
 .../MANIFEST.in                               |  0
 .../README.md                                 |  0
 .../azure/__init__.py                         |  0
 .../azure/eventhub/__init__.py                |  0
 .../azure/eventhub/extensions}/__init__.py    |  0
 .../checkpointerblobaio}/__init__.py          |  0
 .../checkpointerblobaio}/blobstoragepm.py     | 77 +++++++++----------
 .../conftest.py                               |  0
 .../dev_requirements.txt                      |  0
 .../event_processor_blob_storage_example.py   |  8 +-
 .../sdk_packaging.toml                        |  0
 .../setup.cfg                                 |  0
 .../setup.py                                  | 27 +++----
 17 files changed, 53 insertions(+), 69 deletions(-)
 delete mode 100644 sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py
 delete mode 100644 sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/HISTORY.md (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/LICENSE (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/MANIFEST.in (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/README.md (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/azure/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/azure/eventhub/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio => azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions}/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage => azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio}/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage => azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio}/blobstoragepm.py (70%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/conftest.py (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/dev_requirements.txt (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/examples/event_processor_blob_storage_example.py (82%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/sdk_packaging.toml (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/setup.cfg (100%)
 rename sdk/eventhub/{azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage => azure-eventhubs-checkpointerblob-aio}/setup.py (78%)

diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py
deleted file mode 100644
index 62351a0ab30b..000000000000
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py b/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
deleted file mode 100644
index 62351a0ab30b..000000000000
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
+++ /dev/null
@@ -1,5 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/HISTORY.md b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/HISTORY.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/HISTORY.md
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/HISTORY.md
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/LICENSE b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/LICENSE
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/LICENSE
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/LICENSE
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/MANIFEST.in b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/MANIFEST.in
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/MANIFEST.in
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/MANIFEST.in
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/README.md b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/README.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/README.md
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/README.md
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/blobstoragepm.py
similarity index 70%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/blobstoragepm.py
index 1de8b75ac2f7..5c99251f17a6 100644
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/azure/eventhub/aio/eventprocessor/partitionmanagers/blobstorage/blobstoragepm.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/blobstoragepm.py
@@ -6,9 +6,9 @@
 import logging
 from collections import defaultdict
 import asyncio
-from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError
-from azure.core.exceptions import ResourceModifiedError, ResourceExistsError, AzureError
-from azure.storage.blob.aio import ContainerClient
+from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError  # type: ignore
+from azure.core.exceptions import ResourceModifiedError, ResourceExistsError  # type: ignore
+from azure.storage.blob.aio import ContainerClient  # type: ignore
 
 logger = logging.getLogger(__name__)
 UPLOAD_DATA = ""
@@ -30,7 +30,21 @@ def __init__(self, container_client: ContainerClient):
         self._cached_ownership_dict = defaultdict(dict)  # type: Dict[str, Dict[str, Any]]
         # lock each partition for list_ownership, claim_ownership and update_checkpoint etag doesn't get out of sync
         # when the three methods are running concurrently
-        self._cached_ownership_locks = defaultdict(asyncio.Lock)
+        self._cached_ownership_locks = defaultdict(asyncio.Lock)  # type:Dict[str, asyncio.Lock]
+
+    async def _upload_blob(self, ownership, metadata):
+        etag = ownership.get("etag")
+        if etag:
+            etag_match = {"if_match": etag}
+        else:
+            etag_match = {"if_none_match": '*'}
+        partition_id = ownership["partition_id"]
+        blob_client = await self._container_client.upload_blob(
+            name=partition_id, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
+        )
+        uploaded_blob_properties = await blob_client.get_blob_properties()
+        ownership["etag"] = uploaded_blob_properties.etag
+        ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
 
     async def list_ownership(self, eventhub_name: str, consumer_group_name: str) -> Iterable[Dict[str, Any]]:
         try:
@@ -39,24 +53,26 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
             logger.warning("An exception occurred during list_ownership for eventhub %r consumer group %r. "
                            "Exception is %r", eventhub_name, consumer_group_name, err)
             raise
-        async for b in blobs:  # TODO: running them concurrently
+        async for b in blobs:
             async with self._cached_ownership_locks[b.name]:
-                metadata = b.metadata
-                ownership = {
-                    "eventhub_name": eventhub_name,
-                    "consumer_group_name": consumer_group_name,
-                    "partition_id": b.name,
-                    "owner_id": metadata["owner_id"],
-                    "etag": b.etag,
-                    "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
-                }
-                ownership.update(metadata)
-                self._cached_ownership_dict[b.name] = ownership
+                if b.name not in self._cached_ownership_dict \
+                        or b.last_modified.timestamp() > self._cached_ownership_dict[b.name].get("last_modified_time"):
+                    metadata = b.metadata
+                    ownership = {
+                        "eventhub_name": eventhub_name,
+                        "consumer_group_name": consumer_group_name,
+                        "partition_id": b.name,
+                        "owner_id": metadata["owner_id"],
+                        "etag": b.etag,
+                        "last_modified_time": b.last_modified.timestamp() if b.last_modified else None
+                    }
+                    ownership.update(metadata)
+                    self._cached_ownership_dict[b.name] = ownership
         return self._cached_ownership_dict.values()
 
     async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Iterable[Dict[str, Any]]:
         result = []
-        for ownership in ownership_list:  # TODO: claiming concurrently
+        for ownership in ownership_list:
             partition_id = ownership["partition_id"]
             eventhub_name = ownership["eventhub_name"]
             consumer_group_name = ownership["consumer_group_name"]
@@ -68,48 +84,31 @@ async def claim_ownership(self, ownership_list: Iterable[Dict[str, Any]]) -> Ite
                     metadata["offset"] = ownership["offset"]
                 if "sequence_number" in ownership:
                     metadata["sequence_number"] = ownership["sequence_number"]
-
-                etag = ownership.get("etag")
-                if etag:
-                    etag_match = {"if_match": '"'+etag+'"'}
-                else:
-                    etag_match = {"if_none_match": '*'}
                 try:
-                    blob_client = await self._container_client.upload_blob(
-                        name=partition_id, data=UPLOAD_DATA, overwrite=True, metadata=metadata, **etag_match
-                    )
-                    uploaded_blob_properties = await blob_client.get_blob_properties()
-                    ownership["etag"] = uploaded_blob_properties.etag
-                    ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
+                    await self._upload_blob(ownership, metadata)
                     self._cached_ownership_dict[partition_id] = ownership
                     result.append(ownership)
                 except (ResourceModifiedError, ResourceExistsError):
                     logger.info(
                         "EventProcessor instance %r of eventhub %r consumer group %r lost ownership to partition %r",
                         owner_id, eventhub_name, consumer_group_name, partition_id)
-                except Exception as err:
+                except Exception as err:  # pylint:disable=broad-except
                     logger.warning("An exception occurred when EventProcessor instance %r claim_ownership for "
                                    "eventhub %r consumer group %r partition %r. The ownership is now lost. Exception "
                                    "is %r", owner_id, eventhub_name, consumer_group_name, partition_id, err)
-
         return result
 
     async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
                                 offset, sequence_number) -> None:
-
         metadata = {
             "owner_id": owner_id,
             "offset": offset,
             "sequence_number": str(sequence_number)
         }
+        cached_ownership = self._cached_ownership_dict[partition_id]
         async with self._cached_ownership_locks[partition_id]:
             try:
-                blob_client = await self._container_client.upload_blob(
-                    name=partition_id, data=UPLOAD_DATA, metadata=metadata, overwrite=True)
-                uploaded_blob_properties = await blob_client.get_blob_properties()
-                cached_ownership = self._cached_ownership_dict[partition_id]
-                cached_ownership["etag"] = uploaded_blob_properties.etag
-                cached_ownership["last_modified_time"] = uploaded_blob_properties.last_modified.timestamp()
+                await self._upload_blob(cached_ownership, metadata)
             except (ResourceModifiedError, ResourceExistsError):
                 logger.info(
                     "EventProcessor instance %r of eventhub %r consumer group %r couldn't update_checkpoint to "
@@ -119,6 +118,6 @@ async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_
             except Exception as err:
                 logger.warning(
                     "EventProcessor instance %r of eventhub %r consumer group %r couldn't update_checkpoint to "
-                    "partition %r because of unexpected error. Exception is",
+                    "partition %r because of unexpected error. Exception is %r",
                     owner_id, eventhub_name, consumer_group_name, partition_id, err)
                 raise  # EventProcessor will catch the exception and handle it
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/conftest.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/conftest.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/conftest.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/conftest.py
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/dev_requirements.txt b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/dev_requirements.txt
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/dev_requirements.txt
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/dev_requirements.txt
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py
similarity index 82%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py
index e371971f7029..c22b820831d1 100644
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py
@@ -3,7 +3,7 @@
 import os
 from azure.eventhub.aio import EventHubClient
 from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor
-from azure.eventhub.aio.eventprocessor.partitionmanagers.blobstorage import BlobPartitionManager
+from azure.eventhub.extensions.checkpointerblobaio import BlobPartitionManager
 from azure.storage.blob.aio import ContainerClient
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
@@ -20,12 +20,12 @@ async def do_operation(event):
 
 
 class MyPartitionProcessor(PartitionProcessor):
-    async def process_events(self, events, checkpoint_manager):
+    async def process_events(self, events, partition_context):
         if events:
             await asyncio.gather(*[do_operation(event) for event in events])
-            await checkpoint_manager.update_checkpoint(events[-1].offset, events[-1].sequence_number)
+            await partition_context.update_checkpoint(events[-1].offset, events[-1].sequence_number)
         else:
-            print("empty events received", "partition:", checkpoint_manager.partition_id)
+            print("empty events received", "partition:", partition_context.partition_id)
 
 
 if __name__ == '__main__':
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/sdk_packaging.toml
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/sdk_packaging.toml
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/sdk_packaging.toml
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.cfg b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.cfg
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.cfg
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.cfg
diff --git a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py
similarity index 78%
rename from sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py
rename to sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py
index 8f7ce31a18f3..148fb612978e 100644
--- a/sdk/eventhub/azure-eventhubs-aio-eventprocessor-partitionmanagers-blobstorage/setup.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py
@@ -12,15 +12,15 @@
 from io import open
 from setuptools import find_packages, setup
 
+if sys.version_info < (3, 5, 3):
+    raise RuntimeError('Only python 3.5.3 or above is supported')
 
 # Change the PACKAGE_NAME only to change folder and different name
-PACKAGE_NAME = "azure-eventhub-aio-eventprocessor-partitionmanagers-blobstorage"
-PACKAGE_PPRINT_NAME = "Event Hubs Event Procesor Partition Manager implementation with Blob Storage"
+PACKAGE_NAME = "azure-eventhub-checkpointerblob-aio"
+PACKAGE_PPRINT_NAME = "Event Hubs checkpointer implementation with Blob Storage"
 
-# a-b-c => a/b/c
-package_folder_path = PACKAGE_NAME.replace('-', '/')
-# a-b-c => a.b.c
-namespace_name = PACKAGE_NAME.replace('-', '.')
+package_folder_path = "azure/eventhub/extensions/checkpointerblobaio"
+namespace_name = "azure.eventhub.extensions.checkpointerblobaio"
 
 # Version extraction inspired from 'requests'
 with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd:
@@ -40,14 +40,10 @@
         'examples',
         # Exclude packages that will be covered by PEP420 or nspkg
         'azure',
+        'azure.eventhub',
+        'azure.eventhub.extensions',
     ]
 
-if sys.version_info < (3, 5, 3):
-    exclude_packages.extend([
-        '*.aio',
-        '*.aio.*',
-    ])
-
 setup(
     name=PACKAGE_NAME,
     version=version,
@@ -57,12 +53,10 @@
     license='MIT License',
     author='Microsoft Corporation',
     author_email='azpysdkhelp@microsoft.com',
-    url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs',
+    url='https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhubs-checkpointerblob-aio',
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
@@ -73,8 +67,9 @@
     packages=find_packages(exclude=exclude_packages),
     install_requires=[
         'azure-storage-blob<13.0.0,>=12.0.0b2',
+        'azure-eventhub<6.0.0,>=5.0.0b2',
     ],
     extras_require={
-        ":python_version<'3.0'": ['azure-nspkg'],
+
     }
 )

From 8a32e447de702591ea4b20a1f04b4f0f6442303c Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Fri, 6 Sep 2019 09:17:08 -0700
Subject: [PATCH 45/47] replace checkpointer with checkpointstore as a part of
 package name

---
 .../HISTORY.md                                              | 0
 .../LICENSE                                                 | 0
 .../MANIFEST.in                                             | 0
 .../README.md                                               | 0
 .../azure/__init__.py                                       | 0
 .../azure/eventhub/__init__.py                              | 0
 .../azure/eventhub/extensions/__init__.py                   | 0
 .../eventhub/extensions/checkpointstoreblobaio}/__init__.py | 0
 .../extensions/checkpointstoreblobaio}/blobstoragepm.py     | 0
 .../conftest.py                                             | 0
 .../dev_requirements.txt                                    | 0
 .../examples/event_processor_blob_storage_example.py        | 4 ++--
 .../sdk_packaging.toml                                      | 0
 .../setup.cfg                                               | 0
 .../setup.py                                                | 6 +++---
 15 files changed, 5 insertions(+), 5 deletions(-)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/HISTORY.md (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/LICENSE (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/MANIFEST.in (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/README.md (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/azure/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/azure/eventhub/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/azure/eventhub/extensions/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio => azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio}/__init__.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio => azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio}/blobstoragepm.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/conftest.py (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/dev_requirements.txt (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/examples/event_processor_blob_storage_example.py (93%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/sdk_packaging.toml (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/setup.cfg (100%)
 rename sdk/eventhub/{azure-eventhubs-checkpointerblob-aio => azure-eventhubs-checkpointstoreblob-aio}/setup.py (92%)

diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/HISTORY.md b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/HISTORY.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/HISTORY.md
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/HISTORY.md
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/LICENSE b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/LICENSE
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/LICENSE
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/LICENSE
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/MANIFEST.in b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/MANIFEST.in
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/MANIFEST.in
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/MANIFEST.in
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/README.md b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/README.md
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/README.md
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/README.md
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/__init__.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/__init__.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/__init__.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/__init__.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/azure/eventhub/extensions/checkpointerblobaio/blobstoragepm.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/conftest.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/conftest.py
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/conftest.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/conftest.py
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/dev_requirements.txt b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/dev_requirements.txt
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/dev_requirements.txt
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/dev_requirements.txt
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/examples/event_processor_blob_storage_example.py
similarity index 93%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/examples/event_processor_blob_storage_example.py
index c22b820831d1..e7edc047831a 100644
--- a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/examples/event_processor_blob_storage_example.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/examples/event_processor_blob_storage_example.py
@@ -3,7 +3,7 @@
 import os
 from azure.eventhub.aio import EventHubClient
 from azure.eventhub.aio.eventprocessor import EventProcessor, PartitionProcessor
-from azure.eventhub.extensions.checkpointerblobaio import BlobPartitionManager
+from azure.eventhub.extensions.checkpointstoreblobaio import BlobPartitionManager
 from azure.storage.blob.aio import ContainerClient
 
 RECEIVE_TIMEOUT = 5  # timeout in seconds for a receiving operation. 0 or None means no timeout
@@ -33,7 +33,7 @@ async def process_events(self, events, partition_context):
     client = EventHubClient.from_connection_string(CONNECTION_STR, receive_timeout=RECEIVE_TIMEOUT, retry_total=RETRY_TOTAL)
     container_client = ContainerClient.from_connection_string(STORAGE_CONNECTION_STR, container="eventprocessor")
     partition_manager = BlobPartitionManager(container_client=container_client)
-    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=5)
+    event_processor = EventProcessor(client, "$default", MyPartitionProcessor, partition_manager, polling_interval=10)
     try:
         loop.run_until_complete(event_processor.start())
     except KeyboardInterrupt:
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/sdk_packaging.toml b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/sdk_packaging.toml
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/sdk_packaging.toml
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/sdk_packaging.toml
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.cfg b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/setup.cfg
similarity index 100%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.cfg
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/setup.cfg
diff --git a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/setup.py
similarity index 92%
rename from sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py
rename to sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/setup.py
index 148fb612978e..257854d88cb0 100644
--- a/sdk/eventhub/azure-eventhubs-checkpointerblob-aio/setup.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/setup.py
@@ -16,11 +16,11 @@
     raise RuntimeError('Only python 3.5.3 or above is supported')
 
 # Change the PACKAGE_NAME only to change folder and different name
-PACKAGE_NAME = "azure-eventhub-checkpointerblob-aio"
+PACKAGE_NAME = "azure-eventhub-checkpointstoreblob-aio"
 PACKAGE_PPRINT_NAME = "Event Hubs checkpointer implementation with Blob Storage"
 
-package_folder_path = "azure/eventhub/extensions/checkpointerblobaio"
-namespace_name = "azure.eventhub.extensions.checkpointerblobaio"
+package_folder_path = "azure/eventhub/extensions/checkpointstoreblobaio"
+namespace_name = "azure.eventhub.extensions.checkpointstoreblobaio"
 
 # Version extraction inspired from 'requests'
 with open(os.path.join(package_folder_path, '__init__.py'), 'r') as fd:

From 667f0b04d4ab33432c00604824c563548f9e55c9 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Fri, 6 Sep 2019 22:35:04 -0700
Subject: [PATCH 46/47] remove duplicated partition manager

---
 .../partitionmanagers/__init__.py             |  10 --
 .../sample_partition_manager.py               | 144 ------------------
 2 files changed, 154 deletions(-)
 delete mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
 delete mode 100644 sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py

diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
deleted file mode 100644
index d81fee2a97cd..000000000000
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# -----------------------------------------------------------------------------------
-
-from .sample_partition_manager import SamplePartitionManager
-
-__all__ = [
-    'SamplePartitionManager',
-]
diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py
deleted file mode 100644
index 4859d675d5b9..000000000000
--- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/eventprocessor/partitionmanagers/sample_partition_manager.py
+++ /dev/null
@@ -1,144 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# -----------------------------------------------------------------------------------
-
-import time
-import uuid
-import sqlite3
-import logging
-from azure.eventhub.aio.eventprocessor import PartitionManager, OwnershipLostError
-
-logger = logging.getLogger(__name__)
-
-
-def _check_table_name(table_name: str):
-    for c in table_name:
-        if not (c.isalnum() or c == "_"):
-            raise ValueError("Table name \"{}\" is not in correct format".format(table_name))
-    return table_name
-
-
-class SamplePartitionManager(PartitionManager):
-    """An implementation of PartitionManager by using the sqlite3 in Python standard library.
-    Sqlite3 is a mini sql database that runs in memory or files.
-    Please don't use this PartitionManager for production use.
-
-
-    """
-    primary_keys_dict = {"eventhub_name": "text", "consumer_group_name": "text", "partition_id": "text"}
-    other_fields_dict = {"owner_id": "text", "owner_level": "integer", "sequence_number": "integer", "offset": "text",
-                         "last_modified_time": "real", "etag": "text"}
-    checkpoint_fields = ["sequence_number", "offset"]
-    fields_dict = {**primary_keys_dict, **other_fields_dict}
-    primary_keys = list(primary_keys_dict.keys())
-    other_fields = list(other_fields_dict.keys())
-    fields = primary_keys + other_fields
-
-    def __init__(self, db_filename: str = ":memory:", ownership_table: str = "ownership"):
-        """
-
-        :param db_filename: name of file that saves the sql data.
-        Sqlite3 will run in memory without a file when db_filename is ":memory:".
-        :param ownership_table: The table name of the sqlite3 database.
-        """
-        super(SamplePartitionManager, self).__init__()
-        self.ownership_table = _check_table_name(ownership_table)
-        conn = sqlite3.connect(db_filename)
-        c = conn.cursor()
-        try:
-            sql = "create table if not exists " + _check_table_name(ownership_table)\
-                  + "("\
-                  + ",".join([x[0]+" "+x[1] for x in self.fields_dict.items()])\
-                  + ", constraint pk_ownership PRIMARY KEY ("\
-                  + ",".join(self.primary_keys)\
-                  + "))"
-            c.execute(sql)
-        finally:
-            c.close()
-        self.conn = conn
-
-    async def list_ownership(self, eventhub_name, consumer_group_name):
-        cursor = self.conn.cursor()
-        try:
-            cursor.execute("select " + ",".join(self.fields) +
-                                " from "+_check_table_name(self.ownership_table)+" where eventhub_name=? "
-                                "and consumer_group_name=?",
-                                (eventhub_name, consumer_group_name))
-            return [dict(zip(self.fields, row)) for row in cursor.fetchall()]
-        finally:
-            cursor.close()
-
-    async def claim_ownership(self, partitions):
-        result = []
-        cursor = self.conn.cursor()
-        try:
-            for p in partitions:
-                cursor.execute("select etag from " + _check_table_name(self.ownership_table) +
-                                    " where "+ " and ".join([field+"=?" for field in self.primary_keys]),
-                                    tuple(p.get(field) for field in self.primary_keys))
-                cursor_fetch = cursor.fetchall()
-                if not cursor_fetch:
-                    p["last_modified_time"] = time.time()
-                    p["etag"] = str(uuid.uuid4())
-                    try:
-                        fields_without_checkpoint = list(filter(lambda x: x not in self.checkpoint_fields, self.fields))
-                        sql = "insert into " + _check_table_name(self.ownership_table) + " (" \
-                              + ",".join(fields_without_checkpoint) \
-                              + ") values (?,?,?,?,?,?,?)"
-                        cursor.execute(sql, tuple(p.get(field) for field in fields_without_checkpoint))
-                    except sqlite3.OperationalError as op_err:
-                        logger.info("EventProcessor %r failed to claim partition %r "
-                                    "because it was claimed by another EventProcessor at the same time. "
-                                    "The Sqlite3 exception is %r", p["owner_id"], p["partition_id"], op_err)
-                        break
-                    else:
-                        result.append(p)
-                else:
-                    if p.get("etag") == cursor_fetch[0][0]:
-                        p["last_modified_time"] = time.time()
-                        p["etag"] = str(uuid.uuid4())
-                        other_fields_without_checkpoint = list(
-                            filter(lambda x: x not in self.checkpoint_fields, self.other_fields)
-                        )
-                        sql = "update " + _check_table_name(self.ownership_table) + " set "\
-                                       + ','.join([field+"=?" for field in other_fields_without_checkpoint])\
-                                       + " where "\
-                                       + " and ".join([field+"=?" for field in self.primary_keys])
-
-                        cursor.execute(sql, tuple(p.get(field) for field in other_fields_without_checkpoint)
-                                       + tuple(p.get(field) for field in self.primary_keys))
-                        result.append(p)
-                    else:
-                        logger.info("EventProcessor %r failed to claim partition %r "
-                                    "because it was claimed by another EventProcessor at the same time", p["owner_id"],
-                                    p["partition_id"])
-            self.conn.commit()
-            return partitions
-        finally:
-            cursor.close()
-
-    async def update_checkpoint(self, eventhub_name, consumer_group_name, partition_id, owner_id,
-            offset, sequence_number):
-        cursor = self.conn.cursor()
-        try:
-            cursor.execute("select owner_id from " + _check_table_name(self.ownership_table)
-                           + " where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                           (eventhub_name, consumer_group_name, partition_id))
-            cursor_fetch = cursor.fetchall()
-            if cursor_fetch and owner_id == cursor_fetch[0][0]:
-                cursor.execute("update " + _check_table_name(self.ownership_table)
-                               + " set offset=?, sequence_number=? "
-                                 "where eventhub_name=? and consumer_group_name=? and partition_id=?",
-                               (offset, sequence_number, eventhub_name, consumer_group_name, partition_id))
-                self.conn.commit()
-            else:
-                logger.info("EventProcessor couldn't checkpoint to partition %r because it no longer has the ownership",
-                            partition_id)
-                raise OwnershipLostError()
-
-        finally:
-            cursor.close()
-
-    async def close(self):
-        self.conn.close()

From f28365c7332005738e07e9c5f20da66328c0a0d7 Mon Sep 17 00:00:00 2001
From: yijxie <yijun.xie@micrsofot.com>
Date: Fri, 6 Sep 2019 23:27:52 -0700
Subject: [PATCH 47/47] Fix a bug in list_ownership

---
 .../eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py
index 5c99251f17a6..85fcc1ca2eb5 100644
--- a/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py
+++ b/sdk/eventhub/azure-eventhubs-checkpointstoreblob-aio/azure/eventhub/extensions/checkpointstoreblobaio/blobstoragepm.py
@@ -56,7 +56,7 @@ async def list_ownership(self, eventhub_name: str, consumer_group_name: str) ->
         async for b in blobs:
             async with self._cached_ownership_locks[b.name]:
                 if b.name not in self._cached_ownership_dict \
-                        or b.last_modified.timestamp() > self._cached_ownership_dict[b.name].get("last_modified_time"):
+                        or b.last_modified.timestamp() >= self._cached_ownership_dict[b.name].get("last_modified_time"):
                     metadata = b.metadata
                     ownership = {
                         "eventhub_name": eventhub_name,