diff --git a/sdk/storage/azure-storage-file/HISTORY.md b/sdk/storage/azure-storage-file/HISTORY.md index 548bb8b80d3d..542a943e12f5 100644 --- a/sdk/storage/azure-storage-file/HISTORY.md +++ b/sdk/storage/azure-storage-file/HISTORY.md @@ -14,7 +14,6 @@ To use a directory_url, the method `from_directory_url` must be used. - `set_share_access_policy` has required parameter `signed_identifiers`. - NoRetry policy has been removed. Use keyword argument `retry_total=0` for no retries. - Removed types that were accidentally exposed from two modules. Only `FileServiceClient`, `ShareClient`, `DirectoryClient` and `FileClient` should be imported from azure.storage.file.aio -- NoRetry policy has been removed. Use keyword argument `retry_total=0` for no retries. - Some parameters have become keyword only, rather than positional. Some examples include: - `loop` - `max_concurrency` @@ -27,7 +26,12 @@ the following APIs: - upload_range_from_url - clear_range - get_ranges -- `close_handles` renamed to `begin_close_handles` +- `FileClient.close_handles` and `DirectoryClient.close_handles` have both been replaced by two functions each; `close_handle(handle)` and `close_all_handles()`. These functions are blocking and return integers (the number of closed handles) rather than polling objects. + +**New features** + +- `ResourceTypes`, `NTFSAttributes`, and `Services` now have method `from_string` which takes parameters as a string. + ## Version 12.0.0b4: diff --git a/sdk/storage/azure-storage-file/azure/storage/file/_polling.py b/sdk/storage/azure-storage-file/azure/storage/file/_polling.py deleted file mode 100644 index d956616c2a71..000000000000 --- a/sdk/storage/azure-storage-file/azure/storage/file/_polling.py +++ /dev/null @@ -1,66 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import logging -import time -from typing import Any, Callable -from azure.core.polling import PollingMethod - -from ._shared.response_handlers import process_storage_error -from ._generated.models import StorageErrorException - - -logger = logging.getLogger(__name__) - - -class CloseHandles(PollingMethod): - - def __init__(self, interval): - self._command = None - self._continuation_token = None - self._exception = None - self.handles_closed = 0 - self.polling_interval = interval - - def _update_status(self): - try: - status = self._command(marker=self._continuation_token) - except StorageErrorException as error: - process_storage_error(error) - self._continuation_token = status.get('marker') - self.handles_closed += status.get('number_of_handles_closed') or 0 - - def initialize(self, command, initial_status, _): # pylint: disable=arguments-differ - # type: (Any, Any, Callable) -> None - self._command = command - self._continuation_token = initial_status['marker'] - self.handles_closed = initial_status['number_of_handles_closed'] - - def run(self): - # type: () -> None - try: - while not self.finished(): - self._update_status() - time.sleep(self.polling_interval) - except Exception as e: - logger.warning(str(e)) - raise - - def status(self): - self._update_status() - return self.handles_closed - - def finished(self): - # type: () -> bool - """Is this polling finished? - :rtype: bool - """ - return self._continuation_token is None - - def resource(self): - # type: () -> Any - if not self.finished: - self._update_status() - return self.handles_closed diff --git a/sdk/storage/azure-storage-file/azure/storage/file/aio/_polling_async.py b/sdk/storage/azure-storage-file/azure/storage/file/aio/_polling_async.py deleted file mode 100644 index faa60276d1af..000000000000 --- a/sdk/storage/azure-storage-file/azure/storage/file/aio/_polling_async.py +++ /dev/null @@ -1,64 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import asyncio -import logging -from typing import Any, Callable -from azure.core.polling import AsyncPollingMethod - -from .._shared.response_handlers import process_storage_error -from .._generated.models import StorageErrorException - - -logger = logging.getLogger(__name__) - - -class CloseHandlesAsync(AsyncPollingMethod): - - def __init__(self, interval): - self._command = None - self._continuation_token = None - self._exception = None - self.handles_closed = 0 - self.polling_interval = interval - - async def _update_status(self): - try: - status = await self._command(marker=self._continuation_token) - except StorageErrorException as error: - process_storage_error(error) - self._continuation_token = status.get('marker') - self.handles_closed += status.get('number_of_handles_closed') or 0 - - def initialize(self, command, initial_status, _): # pylint: disable=arguments-differ - # type: (Any, Any, Callable) -> None - self._command = command - self._continuation_token = initial_status['marker'] - self.handles_closed = initial_status['number_of_handles_closed'] - - async def run(self): - # type: () -> None - try: - while not self.finished(): - await self._update_status() - await asyncio.sleep(self.polling_interval) - except Exception as e: - logger.warning(str(e)) - raise - - def status(self): - return self.handles_closed - - def finished(self): - # type: () -> bool - """Is this polling finished? - :rtype: bool - """ - return self._continuation_token is None - - def resource(self): - # type: () -> Any - return self.handles_closed diff --git a/sdk/storage/azure-storage-file/azure/storage/file/aio/directory_client_async.py b/sdk/storage/azure-storage-file/azure/storage/file/aio/directory_client_async.py index 63021bca89bc..f809da28baec 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/aio/directory_client_async.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/aio/directory_client_async.py @@ -5,11 +5,11 @@ # -------------------------------------------------------------------------- import functools +import time from typing import ( # pylint: disable=unused-import Optional, Union, Any, Dict, TYPE_CHECKING ) -from azure.core.polling import async_poller from azure.core.async_paging import AsyncItemPaged from azure.core.tracing.decorator import distributed_trace @@ -26,7 +26,6 @@ from .._shared.response_handlers import return_response_headers, process_storage_error from .._deserialize import deserialize_directory_properties from ..directory_client import DirectoryClient as DirectoryClientBase -from ._polling_async import CloseHandlesAsync from .file_client_async import FileClient from .models import DirectoryPropertiesPaged, HandlesPaged @@ -108,7 +107,7 @@ def get_file_client(self, file_name, **kwargs): :param file_name: The name of the file. :returns: A File Client. - :rtype: ~azure.storage.file.file_client.FileClient + :rtype: ~azure.storage.file.FileClient """ if self.directory_path: file_name = self.directory_path.rstrip('/') + "/" + file_name @@ -126,7 +125,7 @@ def get_subdirectory_client(self, directory_name, **kwargs): :param str directory_name: The name of the subdirectory. :returns: A Directory Client. - :rtype: ~azure.storage.file.aio.directory_client_async.DirectoryClient + :rtype: ~azure.storage.file.aio.DirectoryClient .. admonition:: Example: @@ -262,50 +261,79 @@ def list_handles(self, recursive=False, **kwargs): page_iterator_class=HandlesPaged) @distributed_trace_async - async def close_handles( - self, handle=None, # type: Union[str, HandleItem] - recursive=False, # type: bool - **kwargs # type: Any - ): - # type: (...) -> int - """Close open file handles. + async def close_handle(self, handle, **kwargs): + # type: (Union[str, HandleItem], Any) -> int + """Close an open file handle. :param handle: - Optionally, a specific handle to close. The default value is '*' - which will attempt to close all open handles. + A specific handle to close. :type handle: str or ~azure.storage.file.Handle - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified by the client, - its files, its subdirectories and their files. Default value is False. :keyword int timeout: The timeout parameter is expressed in seconds. - :returns: The number of file handles that were closed. + :returns: + The total number of handles closed. This could be 0 if the supplied + handle was not found. :rtype: int """ - timeout = kwargs.pop('timeout', None) try: handle_id = handle.id # type: ignore except AttributeError: - handle_id = handle or '*' - command = functools.partial( - self._client.directory.force_close_handles, - handle_id, - timeout=timeout, - sharesnapshot=self.snapshot, - recursive=recursive, - cls=return_response_headers, - **kwargs) + handle_id = handle + if handle_id == '*': + raise ValueError("Handle ID '*' is not supported. Use 'close_all_handles' instead.") try: - start_close = await command() + response = await self._client.directory.force_close_handles( + handle_id, + marker=None, + recursive=None, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + return response.get('number_of_handles_closed', 0) except StorageErrorException as error: process_storage_error(error) - polling_method = CloseHandlesAsync(self._config.copy_polling_interval) - return await async_poller( - command, - start_close, - None, - polling_method) + @distributed_trace_async + async def close_all_handles(self, recursive=False, **kwargs): + # type: (bool, Any) -> int + """Close any open file handles. + + This operation will block until the service has closed all open handles. + + :param bool recursive: + Boolean that specifies if operation should apply to the directory specified by the client, + its files, its subdirectories and their files. Default value is False. + :keyword int timeout: + The timeout parameter is expressed in seconds. + :returns: The total number of handles closed. + :rtype: int + """ + timeout = kwargs.pop('timeout', None) + start_time = time.time() + + try_close = True + continuation_token = None + total_handles = 0 + while try_close: + try: + response = await self._client.directory.force_close_handles( + handle_id='*', + timeout=timeout, + marker=continuation_token, + recursive=recursive, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + except StorageErrorException as error: + process_storage_error(error) + continuation_token = response.get('marker') + try_close = bool(continuation_token) + total_handles += response.get('number_of_handles_closed', 0) + if timeout: + timeout = max(0, timeout - (time.time() - start_time)) + return total_handles @distributed_trace_async async def get_directory_properties(self, **kwargs): diff --git a/sdk/storage/azure-storage-file/azure/storage/file/aio/file_client_async.py b/sdk/storage/azure-storage-file/azure/storage/file/aio/file_client_async.py index 116aae0ae716..07f09dfc36d6 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/aio/file_client_async.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/aio/file_client_async.py @@ -5,11 +5,11 @@ # -------------------------------------------------------------------------- import functools +import time from io import BytesIO from typing import Optional, Union, IO, List, Dict, Any, Iterable, TYPE_CHECKING # pylint: disable=unused-import import six -from azure.core.polling import async_poller from azure.core.async_paging import AsyncItemPaged from azure.core.tracing.decorator import distributed_trace @@ -27,7 +27,6 @@ from .._shared.response_handlers import return_response_headers, process_storage_error from .._deserialize import deserialize_file_properties, deserialize_file_stream from ..file_client import FileClient as FileClientBase -from ._polling_async import CloseHandlesAsync from .models import HandlesPaged from .download_async import StorageStreamDownloader @@ -864,40 +863,71 @@ def list_handles(self, **kwargs): page_iterator_class=HandlesPaged) @distributed_trace_async - async def close_handles( - self, - handle=None, # type: Union[str, HandleItem] - **kwargs # type: Any - ): - # type: (...) -> int - """Close open file handles. + async def close_handle(self, handle, **kwargs): + # type: (Union[str, HandleItem], Any) -> int + """Close an open file handle. :param handle: - Optionally, a specific handle to close. The default value is '*' - which will attempt to close all open handles. + A specific handle to close. :type handle: str or ~azure.storage.file.Handle :keyword int timeout: The timeout parameter is expressed in seconds. - :returns: The number of file handles that were closed. + :returns: + The total number of handles closed. This could be 0 if the supplied + handle was not found. :rtype: int """ - timeout = kwargs.pop('timeout', None) try: - handle_id = handle.id # type: ignore + handle_id = handle.id # type: ignore except AttributeError: - handle_id = handle or "*" - command = functools.partial( - self._client.file.force_close_handles, - handle_id, - timeout=timeout, - sharesnapshot=self.snapshot, - cls=return_response_headers, - **kwargs - ) + handle_id = handle + if handle_id == '*': + raise ValueError("Handle ID '*' is not supported. Use 'close_all_handles' instead.") try: - start_close = await command() + response = await self._client.file.force_close_handles( + handle_id, + marker=None, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + return response.get('number_of_handles_closed', 0) except StorageErrorException as error: process_storage_error(error) - polling_method = CloseHandlesAsync(self._config.copy_polling_interval) - return await async_poller(command, start_close, None, polling_method) + @distributed_trace_async + async def close_all_handles(self, **kwargs): + # type: (Any) -> int + """Close any open file handles. + + This operation will block until the service has closed all open handles. + + :keyword int timeout: + The timeout parameter is expressed in seconds. + :returns: The total number of handles closed. + :rtype: int + """ + timeout = kwargs.pop('timeout', None) + start_time = time.time() + + try_close = True + continuation_token = None + total_handles = 0 + while try_close: + try: + response = await self._client.file.force_close_handles( + handle_id='*', + timeout=timeout, + marker=continuation_token, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + except StorageErrorException as error: + process_storage_error(error) + continuation_token = response.get('marker') + try_close = bool(continuation_token) + total_handles += response.get('number_of_handles_closed', 0) + if timeout: + timeout = max(0, timeout - (time.time() - start_time)) + return total_handles diff --git a/sdk/storage/azure-storage-file/azure/storage/file/directory_client.py b/sdk/storage/azure-storage-file/azure/storage/file/directory_client.py index 9f02423b52cf..77d8613179f2 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/directory_client.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/directory_client.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- import functools +import time from typing import ( # pylint: disable=unused-import Optional, Union, Any, Dict, TYPE_CHECKING ) @@ -16,7 +17,6 @@ from urllib2 import quote, unquote # type: ignore import six -from azure.core.polling import LROPoller from azure.core.paging import ItemPaged from azure.core.tracing.decorator import distributed_trace @@ -29,7 +29,6 @@ from ._shared.parser import _str from ._parser import _get_file_permission, _datetime_to_str from ._deserialize import deserialize_directory_properties -from ._polling import CloseHandles from .file_client import FileClient from .models import DirectoryPropertiesPaged, HandlesPaged, NTFSAttributes # pylint: disable=unused-import @@ -231,7 +230,7 @@ def get_subdirectory_client(self, directory_name, **kwargs): :param str directory_name: The name of the subdirectory. :returns: A Directory Client. - :rtype: ~azure.storage.file.directory_client.DirectoryClient + :rtype: ~azure.storage.file.DirectoryClient .. admonition:: Example: @@ -367,53 +366,79 @@ def list_handles(self, recursive=False, **kwargs): page_iterator_class=HandlesPaged) @distributed_trace - def begin_close_handles( - self, handle=None, # type: Union[str, HandleItem] - recursive=False, # type: bool - **kwargs # type: Any - ): - # type: (...) -> Any - """Close open file handles. - - This operation may not finish with a single call, so a long-running poller - is returned that can be used to wait until the operation is complete. + def close_handle(self, handle, **kwargs): + # type: (Union[str, HandleItem], Any) -> int + """Close an open file handle. :param handle: - Optionally, a specific handle to close. The default value is '*' - which will attempt to close all open handles. + A specific handle to close. :type handle: str or ~azure.storage.file.Handle - :param bool recursive: - Boolean that specifies if operation should apply to the directory specified by the client, - its files, its subdirectories and their files. Default value is False. :keyword int timeout: The timeout parameter is expressed in seconds. - :returns: A long-running poller to get operation status. - :rtype: ~azure.core.polling.LROPoller + :returns: + The total number of handles closed. This could be 0 if the supplied + handle was not found. + :rtype: int """ - timeout = kwargs.pop('timeout', None) try: handle_id = handle.id # type: ignore except AttributeError: - handle_id = handle or '*' - command = functools.partial( - self._client.directory.force_close_handles, - handle_id, - timeout=timeout, - sharesnapshot=self.snapshot, - recursive=recursive, - cls=return_response_headers, - **kwargs) + handle_id = handle + if handle_id == '*': + raise ValueError("Handle ID '*' is not supported. Use 'close_all_handles' instead.") try: - start_close = command() + response = self._client.directory.force_close_handles( + handle_id, + marker=None, + recursive=None, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + return response.get('number_of_handles_closed', 0) except StorageErrorException as error: process_storage_error(error) - polling_method = CloseHandles(self._config.copy_polling_interval) - return LROPoller( - command, - start_close, - None, - polling_method) + @distributed_trace + def close_all_handles(self, recursive=False, **kwargs): + # type: (bool, Any) -> int + """Close any open file handles. + + This operation will block until the service has closed all open handles. + + :param bool recursive: + Boolean that specifies if operation should apply to the directory specified by the client, + its files, its subdirectories and their files. Default value is False. + :keyword int timeout: + The timeout parameter is expressed in seconds. + :returns: The total number of handles closed. + :rtype: int + """ + timeout = kwargs.pop('timeout', None) + start_time = time.time() + + try_close = True + continuation_token = None + total_handles = 0 + while try_close: + try: + response = self._client.directory.force_close_handles( + handle_id='*', + timeout=timeout, + marker=continuation_token, + recursive=recursive, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + except StorageErrorException as error: + process_storage_error(error) + continuation_token = response.get('marker') + try_close = bool(continuation_token) + total_handles += response.get('number_of_handles_closed', 0) + if timeout: + timeout = max(0, timeout - (time.time() - start_time)) + return total_handles @distributed_trace def get_directory_properties(self, **kwargs): @@ -533,7 +558,7 @@ def create_subdirectory( :keyword int timeout: The timeout parameter is expressed in seconds. :returns: DirectoryClient - :rtype: ~azure.storage.file.directory_client.DirectoryClient + :rtype: ~azure.storage.file.DirectoryClient .. admonition:: Example: diff --git a/sdk/storage/azure-storage-file/azure/storage/file/file_client.py b/sdk/storage/azure-storage-file/azure/storage/file/file_client.py index 4cac6b3d28ff..8348e8be0f18 100644 --- a/sdk/storage/azure-storage-file/azure/storage/file/file_client.py +++ b/sdk/storage/azure-storage-file/azure/storage/file/file_client.py @@ -5,6 +5,7 @@ # -------------------------------------------------------------------------- # pylint: disable=too-many-lines import functools +import time from io import BytesIO from typing import ( # pylint: disable=unused-import Optional, Union, IO, List, Dict, Any, Iterable, @@ -18,7 +19,6 @@ from urllib2 import quote, unquote # type: ignore import six -from azure.core.polling import LROPoller from azure.core.paging import ItemPaged from azure.core.tracing.decorator import distributed_trace @@ -32,7 +32,6 @@ from ._shared.parser import _str from ._parser import _get_file_permission, _datetime_to_str from ._deserialize import deserialize_file_properties, deserialize_file_stream -from ._polling import CloseHandles from .models import HandlesPaged, NTFSAttributes # pylint: disable=unused-import from ._shared_access_signature import FileSharedAccessSignature from .download import StorageStreamDownloader @@ -1112,45 +1111,70 @@ def list_handles(self, **kwargs): page_iterator_class=HandlesPaged) @distributed_trace - def begin_close_handles( - self, handle=None, # type: Union[str, HandleItem] - **kwargs # type: Any - ): - # type: (...) -> Any - """Close open file handles. - - This operation may not finish with a single call, so a long-running poller - is returned that can be used to wait until the operation is complete. + def close_handle(self, handle, **kwargs): + # type: (Union[str, HandleItem], Any) -> int + """Close an open file handle. :param handle: - Optionally, a specific handle to close. The default value is '*' - which will attempt to close all open handles. + A specific handle to close. :type handle: str or ~azure.storage.file.Handle :keyword int timeout: The timeout parameter is expressed in seconds. - :returns: A long-running poller to get operation status. - :rtype: ~azure.core.polling.LROPoller + :returns: + The number of handles closed (this may be 0 if the specified handle was not found). + :rtype: int """ - timeout = kwargs.pop('timeout', None) try: handle_id = handle.id # type: ignore except AttributeError: - handle_id = handle or '*' - command = functools.partial( - self._client.file.force_close_handles, - handle_id, - timeout=timeout, - sharesnapshot=self.snapshot, - cls=return_response_headers, - **kwargs) + handle_id = handle + if handle_id == '*': + raise ValueError("Handle ID '*' is not supported. Use 'close_all_handles' instead.") try: - start_close = command() + response = self._client.file.force_close_handles( + handle_id, + marker=None, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + return response.get('number_of_handles_closed', 0) except StorageErrorException as error: process_storage_error(error) - polling_method = CloseHandles(self._config.copy_polling_interval) - return LROPoller( - command, - start_close, - None, - polling_method) + @distributed_trace + def close_all_handles(self, **kwargs): + # type: (Any) -> int + """Close any open file handles. + + This operation will block until the service has closed all open handles. + + :keyword int timeout: + The timeout parameter is expressed in seconds. + :returns: The total number of handles closed. + :rtype: int + """ + timeout = kwargs.pop('timeout', None) + start_time = time.time() + + try_close = True + continuation_token = None + total_handles = 0 + while try_close: + try: + response = self._client.file.force_close_handles( + handle_id='*', + timeout=timeout, + marker=continuation_token, + sharesnapshot=self.snapshot, + cls=return_response_headers, + **kwargs + ) + except StorageErrorException as error: + process_storage_error(error) + continuation_token = response.get('marker') + try_close = bool(continuation_token) + total_handles += response.get('number_of_handles_closed', 0) + if timeout: + timeout = max(0, timeout - (time.time() - start_time)) + return total_handles diff --git a/sdk/storage/azure-storage-file/tests/test_handle.py b/sdk/storage/azure-storage-file/tests/test_handle.py index 1aaa7d955909..bbd4c295d633 100644 --- a/sdk/storage/azure-storage-file/tests/test_handle.py +++ b/sdk/storage/azure-storage-file/tests/test_handle.py @@ -186,10 +186,13 @@ def test_close_single_handle(self): self._validate_handles(handles) # Act - num_closed = root.begin_close_handles(handle=handles[0]) + with self.assertRaises(ValueError): + root.close_handle('*') + + num_closed = root.close_handle(handles[0]) # Assert 1 handle has been closed - self.assertEqual(1, num_closed.result()) + self.assertEqual(1, num_closed) @record def test_close_all_handle(self): @@ -205,11 +208,10 @@ def test_close_all_handle(self): self._validate_handles(handles) # Act - num_closed = root.begin_close_handles() - total_num_handle_closed = num_closed.result() + num_closed = root.close_all_handles() # Assert at least 1 handle has been closed - self.assertTrue(total_num_handle_closed > 1) + self.assertTrue(num_closed > 1) # ------------------------------------------------------------------------------