From 1a81a9a29e61df9849b672545f2dd437d8ed1db3 Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Fri, 11 Dec 2020 07:04:12 +1300
Subject: [PATCH 1/6] First pass

---
 .../tests/perfstress_tests/__init__.py        |  0
 .../tests/perfstress_tests/_test_base.py      | 85 +++++++++++++++++++
 .../tests/perfstress_tests/append.py          | 43 ++++++++++
 .../tests/perfstress_tests/read.py            | 33 +++++++
 .../tests/perfstress_tests/upload.py          | 36 ++++++++
 .../perfstress_tests/upload_from_file.py      | 42 +++++++++
 6 files changed, 239 insertions(+)
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/__init__.py
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py

diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/__init__.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
new file mode 100644
index 000000000000..77500688eb6f
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
@@ -0,0 +1,85 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import uuid
+
+from azure_devtools.perfstress_tests import PerfStressTest
+
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.filedatalake import DataLakeServiceClient as SyncDataLakeServiceClient
+from azure.storage.filedatalake.aio import DataLakeServiceClient as AsyncDataLakeServiceClient
+
+
+class _ServiceTest(PerfStressTest):
+    service_client = None
+    async_service_client = None
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING")
+        if not _ServiceTest.service_client or self.args.no_client_share:
+            _ServiceTest.service_client = SyncDataLakeServiceClient.from_connection_string(conn_str=connection_string)
+            _ServiceTest.async_service_client = AsyncDataLakeServiceClient.from_connection_string(conn_str=connection_string)
+        self.service_client = _ServiceTest.service_client
+        self.async_service_client =_ServiceTest.async_service_client
+
+    async def close(self):
+        await self.async_service_client.close()
+        await super().close()
+
+    @staticmethod
+    def add_arguments(parser):
+        super(_ServiceTest, _ServiceTest).add_arguments(parser)
+        parser.add_argument('-c', '--max-concurrency', nargs='?', type=int, help='Maximum number of concurrent threads used for data transfer. Defaults to 1', default=1)
+        parser.add_argument('-s', '--size', nargs='?', type=int, help='Size of data to transfer.  Default is 10240.', default=10240)
+        parser.add_argument('--no-client-share', action='store_true', help='Create one ServiceClient per test instance.  Default is to share a single ServiceClient.', default=False)
+
+
+class _FileSystemTest(_ServiceTest):
+    fs_name = "perfstress-" + str(uuid.uuid4())
+
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.fs_client = self.service_client.get_file_system_client(self.fs_name)
+        self.async_fs_client = self.async_service_client.get_file_system_client(self.fs_name)
+
+    async def global_setup(self):
+        await super().global_setup()
+        await self.async_fs_client.create_file_system()
+
+    async def global_cleanup(self):
+        await self.async_fs_client.delete_file_system()
+        await super().global_cleanup()
+
+    async def close(self):
+        await self.async_fs_client.close()
+        await super().close()
+
+
+class _FileTest(_FileSystemTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        file_name = "sharefiletest-" + str(uuid.uuid4())
+        self.file_client = self.fs_client.get_file_client(file_name)
+        self.async_file_client = self.async_fs_client.get_file_client(file_name)
+
+    async def global_setup(self):
+        await super().global_setup()
+        try:
+            await self.async_file_client.delete_file()
+        except ResourceNotFoundError:
+            pass
+
+    async def global_cleanup(self):
+        try:
+            await self.async_file_client.delete_file()
+        except ResourceNotFoundError:
+            pass
+        await super().global_cleanup()
+
+    async def close(self):
+        await self.async_file_client.close()
+        await super().close()
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
new file mode 100644
index 000000000000..a4b7e5cf194e
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
@@ -0,0 +1,43 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import uuid
+
+from azure_devtools.perfstress_tests import get_random_bytes, RandomStream, AsyncRandomStream
+
+from ._test_base import _FileSystemTest
+
+
+class AppendTest(_FileSystemTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.data = get_random_bytes(self.args.size)
+        file_name = "filetest-" + str(uuid.uuid4())
+        self.file_client = self.fs_client.get_file_client(file_name)
+        self.async_file_client = self.async_fs_client.get_file_client(file_name)
+
+    async def setup(self):
+        await self.async_file_client.create_file()
+
+    def run_sync(self):
+        data = RandomStream(self.args.size) if self.args.stream else self.data
+        self.file_client.append_data(
+            data,
+            length=self.args.size,
+            offset=0,
+            max_concurrency=self.args.max_concurrency)
+
+    async def run_async(self):
+        data = AsyncRandomStream(self.args.size) if self.args.stream else self.data
+        await self.async_file_client.append_data(
+            data,
+            length=self.args.size,
+            offset=0,
+            max_concurrency=self.args.max_concurrency)
+
+    @staticmethod
+    def add_arguments(parser):
+        super(AppendTest, AppendTest).add_arguments(parser)
+        parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False)
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
new file mode 100644
index 000000000000..59b12bed1ee1
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
@@ -0,0 +1,33 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure_devtools.perfstress_tests import get_random_bytes
+
+from ._test_base import _FileSystemTest
+
+
+class DownloadTest(_FileSystemTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        file_name = "downloadtest"
+        self.file_client = self.fs_client.get_file_client(file_name)
+        self.async_file_client = self.async_fs_client.get_file_client(file_name)
+
+    async def global_setup(self):
+        await super().global_setup()
+        data = get_random_bytes(self.args.size)
+        await self.async_file_client.upload_data(data)
+
+    def run_sync(self):
+        stream = self.file_client.download_file(max_concurrency=self.args.max_concurrency)
+        stream.readall()
+
+    async def run_async(self):
+        stream = await self.async_file_client.download_file(max_concurrency=self.args.max_concurrency)
+        await stream.readall()
+
+    async def close(self):
+        await self.async_file_client.close()
+        await super().close()
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
new file mode 100644
index 000000000000..6a2f8b6ab730
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
@@ -0,0 +1,36 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from ._test_base import _FileTest
+
+from azure_devtools.perfstress_tests import RandomStream, get_random_bytes
+from azure_devtools.perfstress_tests import AsyncRandomStream
+
+
+class UploadTest(_FileTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.data = get_random_bytes(self.args.size)
+
+    def run_sync(self):
+        data = RandomStream(self.args.size) if self.args.stream else self.data
+        self.file_client.upload_data(
+            data,
+            length=self.args.size,
+            overwrite=True,
+            max_concurrency=self.args.max_concurrency)
+
+    async def run_async(self):
+        data = AsyncRandomStream(self.args.size) if self.args.stream else self.data
+        await self.async_file_client.upload_data(
+            data,
+            length=self.args.size,
+            overwrite=True,
+            max_concurrency=self.args.max_concurrency)
+
+    @staticmethod
+    def add_arguments(parser):
+        super(UploadTest, UploadTest).add_arguments(parser)
+        parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False)
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py
new file mode 100644
index 000000000000..e4888ab065f1
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py
@@ -0,0 +1,42 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import os
+import tempfile
+
+from azure_devtools.perfstress_tests import get_random_bytes
+
+from ._test_base import _FileTest
+
+
+class UploadFromFileTest(_FileTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.temp_file = None
+        self.data = get_random_bytes(self.args.size)
+
+    async def global_setup(self):
+        await super().global_setup()
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            self.temp_file = temp_file.name
+            temp_file.write(self.data)
+
+    async def global_cleanup(self):
+        os.remove(self.temp_file)
+        await super().global_cleanup()
+
+    def run_sync(self):
+        with open(self.temp_file) as fp:
+            self.file_client.upload_data(
+                fp,
+                overwrite=True,
+                max_concurrency=self.args.max_concurrency)
+
+    async def run_async(self):
+        with open(self.temp_file) as fp:
+            await self.async_file_client.upload_data(
+                fp,
+                overwrite=True,
+                max_concurrency=self.args.max_concurrency)

From 5f4767f875975802f33d696d8fa2181fc5a0b6d3 Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Thu, 17 Dec 2020 20:43:56 +1300
Subject: [PATCH 2/6] Fixed up tests

---
 .../tests/perfstress_tests/append.py          | 18 +++------
 .../tests/perfstress_tests/read.py            | 11 ++++--
 .../tests/perfstress_tests/upload.py          | 14 ++-----
 .../perfstress_tests/upload_from_file.py      | 16 ++++----
 .../perfstress_tests/__init__.py              |  3 +-
 .../perfstress_tests/async_random_stream.py   | 12 ++++++
 .../perfstress_tests/random_stream.py         | 37 ++++++++++++++++++-
 7 files changed, 71 insertions(+), 40 deletions(-)

diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
index a4b7e5cf194e..46430e4608dc 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
@@ -5,7 +5,7 @@
 
 import uuid
 
-from azure_devtools.perfstress_tests import get_random_bytes, RandomStream, AsyncRandomStream
+from azure_devtools.perfstress_tests import RandomStream, AsyncRandomStream
 
 from ._test_base import _FileSystemTest
 
@@ -13,7 +13,6 @@
 class AppendTest(_FileSystemTest):
     def __init__(self, arguments):
         super().__init__(arguments)
-        self.data = get_random_bytes(self.args.size)
         file_name = "filetest-" + str(uuid.uuid4())
         self.file_client = self.fs_client.get_file_client(file_name)
         self.async_file_client = self.async_fs_client.get_file_client(file_name)
@@ -22,22 +21,15 @@ async def setup(self):
         await self.async_file_client.create_file()
 
     def run_sync(self):
-        data = RandomStream(self.args.size) if self.args.stream else self.data
+        data = RandomStream(self.args.size)
         self.file_client.append_data(
             data,
             length=self.args.size,
-            offset=0,
-            max_concurrency=self.args.max_concurrency)
+            offset=0)
 
     async def run_async(self):
-        data = AsyncRandomStream(self.args.size) if self.args.stream else self.data
+        data = AsyncRandomStream(self.args.size)
         await self.async_file_client.append_data(
             data,
             length=self.args.size,
-            offset=0,
-            max_concurrency=self.args.max_concurrency)
-
-    @staticmethod
-    def add_arguments(parser):
-        super(AppendTest, AppendTest).add_arguments(parser)
-        parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False)
+            offset=0)
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
index 59b12bed1ee1..6812eb06f169 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from azure_devtools.perfstress_tests import get_random_bytes
+from azure_devtools.perfstress_tests import get_random_bytes, WriteStream
 
 from ._test_base import _FileSystemTest
 
@@ -18,15 +18,18 @@ def __init__(self, arguments):
     async def global_setup(self):
         await super().global_setup()
         data = get_random_bytes(self.args.size)
-        await self.async_file_client.upload_data(data)
+        await self.async_file_client.create_file()
+        await self.async_file_client.upload_data(data, overwrite=True)
 
     def run_sync(self):
+        download = WriteStream()
         stream = self.file_client.download_file(max_concurrency=self.args.max_concurrency)
-        stream.readall()
+        stream.readinto(download)
 
     async def run_async(self):
+        download = WriteStream()
         stream = await self.async_file_client.download_file(max_concurrency=self.args.max_concurrency)
-        await stream.readall()
+        await stream.readinto(download)
 
     async def close(self):
         await self.async_file_client.close()
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
index 6a2f8b6ab730..b45b25b953c1 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
@@ -5,17 +5,14 @@
 
 from ._test_base import _FileTest
 
-from azure_devtools.perfstress_tests import RandomStream, get_random_bytes
+from azure_devtools.perfstress_tests import RandomStream
 from azure_devtools.perfstress_tests import AsyncRandomStream
 
 
 class UploadTest(_FileTest):
-    def __init__(self, arguments):
-        super().__init__(arguments)
-        self.data = get_random_bytes(self.args.size)
 
     def run_sync(self):
-        data = RandomStream(self.args.size) if self.args.stream else self.data
+        data = RandomStream(self.args.size)
         self.file_client.upload_data(
             data,
             length=self.args.size,
@@ -23,14 +20,9 @@ def run_sync(self):
             max_concurrency=self.args.max_concurrency)
 
     async def run_async(self):
-        data = AsyncRandomStream(self.args.size) if self.args.stream else self.data
+        data = AsyncRandomStream(self.args.size)
         await self.async_file_client.upload_data(
             data,
             length=self.args.size,
             overwrite=True,
             max_concurrency=self.args.max_concurrency)
-
-    @staticmethod
-    def add_arguments(parser):
-        super(UploadTest, UploadTest).add_arguments(parser)
-        parser.add_argument('--stream', action='store_true', help='Upload stream instead of byte array.', default=False)
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py
index e4888ab065f1..00370b1d1e41 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload_from_file.py
@@ -12,30 +12,28 @@
 
 
 class UploadFromFileTest(_FileTest):
-    def __init__(self, arguments):
-        super().__init__(arguments)
-        self.temp_file = None
-        self.data = get_random_bytes(self.args.size)
+    temp_file = None
 
     async def global_setup(self):
         await super().global_setup()
+        data = get_random_bytes(self.args.size)
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
-            self.temp_file = temp_file.name
-            temp_file.write(self.data)
+            UploadFromFileTest.temp_file = temp_file.name
+            temp_file.write(data)
 
     async def global_cleanup(self):
-        os.remove(self.temp_file)
+        os.remove(UploadFromFileTest.temp_file)
         await super().global_cleanup()
 
     def run_sync(self):
-        with open(self.temp_file) as fp:
+        with open(UploadFromFileTest.temp_file, 'rb') as fp:
             self.file_client.upload_data(
                 fp,
                 overwrite=True,
                 max_concurrency=self.args.max_concurrency)
 
     async def run_async(self):
-        with open(self.temp_file) as fp:
+        with open(UploadFromFileTest.temp_file, 'rb') as fp:
             await self.async_file_client.upload_data(
                 fp,
                 overwrite=True,
diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py
index e3c21008314b..0716333ea00a 100644
--- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py
+++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/__init__.py
@@ -8,13 +8,14 @@
 
 from .perf_stress_runner import PerfStressRunner
 from .perf_stress_test import PerfStressTest
-from .random_stream import RandomStream, get_random_bytes
+from .random_stream import RandomStream, WriteStream, get_random_bytes
 from .async_random_stream import AsyncRandomStream
 
 __all__ = [
     "PerfStressRunner",
     "PerfStressTest",
     "RandomStream",
+    "WriteStream",
     "AsyncRandomStream",
     "get_random_bytes"
 ]
diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
index 11f0d663c416..1dd43dad4045 100644
--- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
+++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
@@ -30,8 +30,20 @@ def read(self, size=None):
             self._base_data = get_random_bytes(e)
             self._base_data_length = e
         self._remaining = self._remaining - e
+        self._position += e
         return self._base_data[:e]
 
+    def seek(self, index, whence=0):
+        if whence == 0:
+            self._position = index
+        elif whence == 1:
+            self._position = self._position + index
+        elif whence == 2:
+            self._position = self._length - 1 + index
+
+    def tell(self):
+        return self._position
+
     def remaining(self):
         return self._remaining
 
diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
index 0ab5283c288f..18534a4437e1 100644
--- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
+++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
@@ -15,10 +15,11 @@ def __init__(self, length, initial_buffer_length=1024*1024):
         self._base_data_length = initial_buffer_length
         self._position = 0
         self._remaining = length
+        self._length = length
 
     def read(self, size=None):
         if self._remaining == 0:
-            return None
+            return b""
 
         if size is None:
             e = self._base_data_length
@@ -29,7 +30,39 @@ def read(self, size=None):
             self._base_data = get_random_bytes(e)
             self._base_data_length = e
         self._remaining = self._remaining - e
+        self._position += e
         return self._base_data[:e]
 
+    def tell(self):
+        return self._position
+
+    def seek(self, index, whence=0):
+        if whence == 0:
+            self._position = index
+        elif whence == 1:
+            self._position = self._position + index
+        elif whence == 2:
+            self._position = self._length - 1 + index
+
     def remaining(self):
-        return self._remaining
\ No newline at end of file
+        return self._remaining
+
+
+class WriteStream:
+
+    def __init__(self):
+        self._position = 0
+
+    def write(self, content):
+        length = len(content)
+        self._position += length
+        return length
+
+    def seek(self, index):
+        self._position = index
+
+    def seekable(self):
+        return True
+
+    def tell(self):
+        return self._position

From 3b9ff19a6cfbd759c6052ab5c1ed9cc741805ad1 Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Mon, 11 Jan 2021 10:36:01 +1300
Subject: [PATCH 3/6] Fixed global setup/cleanup

---
 .../tests/perfstress_tests/_test_base.py           | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
index 77500688eb6f..cf8728fb9cb1 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/_test_base.py
@@ -66,20 +66,6 @@ def __init__(self, arguments):
         self.file_client = self.fs_client.get_file_client(file_name)
         self.async_file_client = self.async_fs_client.get_file_client(file_name)
 
-    async def global_setup(self):
-        await super().global_setup()
-        try:
-            await self.async_file_client.delete_file()
-        except ResourceNotFoundError:
-            pass
-
-    async def global_cleanup(self):
-        try:
-            await self.async_file_client.delete_file()
-        except ResourceNotFoundError:
-            pass
-        await super().global_cleanup()
-
     async def close(self):
         await self.async_file_client.close()
         await super().close()

From fbdbf63c788b92ac1d70b69ef56bbdd41eb6e49e Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Tue, 12 Jan 2021 13:55:57 +1300
Subject: [PATCH 4/6] Reset streams

---
 .../tests/perfstress_tests/append.py          | 10 ++++---
 .../tests/perfstress_tests/read.py            |  9 +++---
 .../tests/perfstress_tests/upload.py          | 12 +++++---
 .../perfstress_tests/async_random_stream.py   | 18 ++++++++----
 .../perfstress_tests/random_stream.py         | 28 ++++++++++++++-----
 5 files changed, 52 insertions(+), 25 deletions(-)

diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
index 46430e4608dc..a12ba507e91a 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/append.py
@@ -16,20 +16,22 @@ def __init__(self, arguments):
         file_name = "filetest-" + str(uuid.uuid4())
         self.file_client = self.fs_client.get_file_client(file_name)
         self.async_file_client = self.async_fs_client.get_file_client(file_name)
+        self.upload_stream = RandomStream(self.args.size)
+        self.upload_stream_async = AsyncRandomStream(self.args.size)
 
     async def setup(self):
         await self.async_file_client.create_file()
 
     def run_sync(self):
-        data = RandomStream(self.args.size)
+        self.upload_stream.reset()
         self.file_client.append_data(
-            data,
+            self.upload_stream,
             length=self.args.size,
             offset=0)
 
     async def run_async(self):
-        data = AsyncRandomStream(self.args.size)
+        self.upload_stream_async.reset()
         await self.async_file_client.append_data(
-            data,
+            self.upload_stream_async,
             length=self.args.size,
             offset=0)
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
index 6812eb06f169..e0addee7b440 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/read.py
@@ -14,6 +14,7 @@ def __init__(self, arguments):
         file_name = "downloadtest"
         self.file_client = self.fs_client.get_file_client(file_name)
         self.async_file_client = self.async_fs_client.get_file_client(file_name)
+        self.download_stream = WriteStream()
 
     async def global_setup(self):
         await super().global_setup()
@@ -22,14 +23,14 @@ async def global_setup(self):
         await self.async_file_client.upload_data(data, overwrite=True)
 
     def run_sync(self):
-        download = WriteStream()
+        self.download_stream.reset()
         stream = self.file_client.download_file(max_concurrency=self.args.max_concurrency)
-        stream.readinto(download)
+        stream.readinto(self.download_stream)
 
     async def run_async(self):
-        download = WriteStream()
+        self.download_stream.reset()
         stream = await self.async_file_client.download_file(max_concurrency=self.args.max_concurrency)
-        await stream.readinto(download)
+        await stream.readinto(self.download_stream)
 
     async def close(self):
         await self.async_file_client.close()
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
index b45b25b953c1..09af7ba6cb4f 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/upload.py
@@ -10,19 +10,23 @@
 
 
 class UploadTest(_FileTest):
+    def __init__(self, arguments):
+        super().__init__(arguments)
+        self.upload_stream = RandomStream(self.args.size)
+        self.upload_stream_async = AsyncRandomStream(self.args.size)
 
     def run_sync(self):
-        data = RandomStream(self.args.size)
+        self.upload_stream.reset()
         self.file_client.upload_data(
-            data,
+            self.upload_stream,
             length=self.args.size,
             overwrite=True,
             max_concurrency=self.args.max_concurrency)
 
     async def run_async(self):
-        data = AsyncRandomStream(self.args.size)
+        self.upload_stream_async.reset()
         await self.async_file_client.upload_data(
-            data,
+            self.upload_stream_async,
             length=self.args.size,
             overwrite=True,
             max_concurrency=self.args.max_concurrency)
diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
index 1dd43dad4045..473b0ac348df 100644
--- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
+++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/async_random_stream.py
@@ -5,30 +5,36 @@
 
 from io import BytesIO
 
-from .random_stream import get_random_bytes
+from .random_stream import get_random_bytes, _DEFAULT_LENGTH
 
 
 class AsyncRandomStream(BytesIO):
-    def __init__(self, length, initial_buffer_length=1024 * 1024):
+    def __init__(self, length, initial_buffer_length=_DEFAULT_LENGTH):
         super().__init__()
         self._base_data = get_random_bytes(initial_buffer_length)
-        self._base_data_length = initial_buffer_length
+        self._data_length = length
+        self._base_buffer_length = initial_buffer_length
         self._position = 0
         self._remaining = length
         self._closed = False
+    
+    def reset(self):
+        self._position = 0
+        self._remaining = self._data_length
+        self._closed = False
 
     def read(self, size=None):
         if self._remaining == 0:
             return b""
 
         if size is None:
-            e = self._base_data_length
+            e = self._base_buffer_length
         else:
             e = size
         e = min(e, self._remaining)
-        if e > self._base_data_length:
+        if e > self._base_buffer_length:
             self._base_data = get_random_bytes(e)
-            self._base_data_length = e
+            self._base_buffer_length = e
         self._remaining = self._remaining - e
         self._position += e
         return self._base_data[:e]
diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
index 18534a4437e1..324a3eb67553 100644
--- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
+++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/random_stream.py
@@ -5,30 +5,41 @@
 
 import os
 
+_DEFAULT_LENGTH = 1024*1024
+_BYTE_BUFFER = [_DEFAULT_LENGTH, os.urandom(_DEFAULT_LENGTH)]
+
+
 def get_random_bytes(buffer_length):
-    return os.urandom(buffer_length)
+    if buffer_length > _BYTE_BUFFER[0]:
+        _BYTE_BUFFER[0] = buffer_length
+        _BYTE_BUFFER[1] = os.urandom(buffer_length)
+    return _BYTE_BUFFER[1][:buffer_length]
 
 
 class RandomStream:
-    def __init__(self, length, initial_buffer_length=1024*1024):
+    def __init__(self, length, initial_buffer_length=_DEFAULT_LENGTH):
         self._base_data = get_random_bytes(initial_buffer_length)
-        self._base_data_length = initial_buffer_length
+        self._data_length = length
+        self._base_buffer_length = initial_buffer_length
         self._position = 0
         self._remaining = length
-        self._length = length
+    
+    def reset(self):
+        self._position = 0
+        self._remaining = self._data_length
 
     def read(self, size=None):
         if self._remaining == 0:
             return b""
 
         if size is None:
-            e = self._base_data_length
+            e = self._base_buffer_length
         else:
             e = size
         e = min(e, self._remaining)
-        if e > self._base_data_length:
+        if e > self._base_buffer_length:
             self._base_data = get_random_bytes(e)
-            self._base_data_length = e
+            self._base_buffer_length = e
         self._remaining = self._remaining - e
         self._position += e
         return self._base_data[:e]
@@ -52,6 +63,9 @@ class WriteStream:
 
     def __init__(self):
         self._position = 0
+    
+    def reset(self):
+        self._position = 0
 
     def write(self, content):
         length = len(content)

From 2f88847b9f2311f0bbec0668912efb986e313f84 Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Tue, 19 Jan 2021 08:25:46 +1300
Subject: [PATCH 5/6] Added perftest readme

---
 eng/.docsettings.yml                          |  1 +
 .../tests/perfstress_tests/README.md          | 49 +++++++++++++++++++
 2 files changed, 50 insertions(+)
 create mode 100644 sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md

diff --git a/eng/.docsettings.yml b/eng/.docsettings.yml
index 07f073a72430..23b1fe73b38c 100644
--- a/eng/.docsettings.yml
+++ b/eng/.docsettings.yml
@@ -8,6 +8,7 @@ omitted_paths:
   - doc/*
   - sdk/**/samples/*
   - sdk/identity/azure-identity/tests/*
+  - sdk/**/tests/perfstress_tests/*
 
 language: python
 root_check_enabled: True
diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md
new file mode 100644
index 000000000000..ffeb096ba9d2
--- /dev/null
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md
@@ -0,0 +1,49 @@
+# DataLake Performance Tests
+
+In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`.
+Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7.
+Note that there are no T1 tests for this project.
+
+### Setup for T2 perf test runs
+
+```cmd
+(env) ~/azure-storage-file-datalake> pip install -r dev_requirements.txt
+(env) ~/azure-storage-file-datalake> pip install -e .
+```
+
+## Test commands
+
+When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature).
+
+```cmd
+(env) ~/azure-storage-file-datalake> cd tests
+(env) ~/azure-storage-file-datalake/tests> perfstress
+```
+Using the `perfstress` command alone will list the available perf tests found.
+
+### Common perf command line options
+These options are available for all perf tests:
+- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10.
+- `--iterations=1` Number of test iterations to run. Default is 1.
+- `--parallel=1` Number of tests to run in parallel. Default is 1.
+- `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing).
+- `--warm-up=5` Number of seconds to spend warming up the connection before measuing begins. Default is 5.
+- `--sync` Whether to run the tests in sync or async. Default is False (async).
+- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted).
+
+### Common DataLake command line options
+The options are available for all SB perf tests:
+- `--size=10240` Size in bytes of data to be transferred in upload or download tests. Default is 10240.
+- `--max-concurrency=1` Number of threads to concurrently upload/download a single operation using the SDK API parameter. Default is 1.
+
+### T2 Tests
+The tests currently written for the T2 SDK:
+- `UploadTest` Uploads a stream of `size` bytes to a new File.
+- `UploadFromFileTest` Uploads a local file of `size` bytes to a new File.
+- `DownloadTest` Download a stream of `size` bytes. 
+- `AppendTest` Append `size` bytes to an existing file.
+
+## Example command
+```cmd
+(env) ~/azure-storage-file-datalake/tests> perfstress UploadTest --parallel=2 --size=10240
+```
\ No newline at end of file

From f1508dde4dee07c8879e4e032ae75c5426dd4600 Mon Sep 17 00:00:00 2001
From: antisch <antisch@microsoft.com>
Date: Thu, 4 Mar 2021 07:30:45 +1300
Subject: [PATCH 6/6] Update readme

---
 .../tests/perfstress_tests/README.md                     | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md
index ffeb096ba9d2..dbd350bfe1e5 100644
--- a/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md
+++ b/sdk/storage/azure-storage-file-datalake/tests/perfstress_tests/README.md
@@ -4,6 +4,13 @@ In order to run the performance tests, the `azure-devtools` package must be inst
 Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7.
 Note that there are no T1 tests for this project.
 
+### Setup for test resources
+
+These tests will run against a pre-configured Storage account. The following environment variable will need to be set for the tests to access the live resources:
+```
+AZURE_STORAGE_CONNECTION_STRING=<live storage account connection string>
+```
+
 ### Setup for T2 perf test runs
 
 ```cmd
@@ -27,7 +34,7 @@ These options are available for all perf tests:
 - `--iterations=1` Number of test iterations to run. Default is 1.
 - `--parallel=1` Number of tests to run in parallel. Default is 1.
 - `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing).
-- `--warm-up=5` Number of seconds to spend warming up the connection before measuing begins. Default is 5.
+- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5.
 - `--sync` Whether to run the tests in sync or async. Default is False (async).
 - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted).