diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py index de40ea7ca058..0260537a02bb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py @@ -1 +1 @@ -__import__('pkg_resources').declare_namespace(__name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py index de40ea7ca058..0260537a02bb 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py @@ -1 +1 @@ -__import__('pkg_resources').declare_namespace(__name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py index b450d83e430d..062a035662e3 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py @@ -5,10 +5,10 @@ # -------------------------------------------------------------------------- __author__ = 'Microsoft Corp. ' -__version__ = '1.3.1' +__version__ = '2.0.1' # x-ms-version for storage service. -X_MS_VERSION = '2018-03-28' +X_MS_VERSION = '2018-11-09' # internal configurations, should not be changed _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024 diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py index 3365ebfa726c..969f256b4a76 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py @@ -10,7 +10,6 @@ from xml.etree import cElementTree as ETree except ImportError: from xml.etree import ElementTree as ETree - from ..common._common_conversion import ( _decode_base64_to_text, _to_str, @@ -37,6 +36,7 @@ ResourceProperties, BlobPrefix, AccountInformation, + UserDelegationKey, ) from ._encryption import _decrypt_blob from ..common.models import _list @@ -352,6 +352,77 @@ def _convert_xml_to_blob_list(response): return blob_list +def _convert_xml_to_blob_name_list(response): + ''' + + + string-value + string-value + int-value + string-value + + + blob-name + true + date-time-value + + date-time-value + etag + size-in-bytes + blob-content-type + + + + + sequence-number + BlockBlob|PageBlob|AppendBlob + locked|unlocked + available | leased | expired | breaking | broken + infinite | fixed + id + pending | success | aborted | failed + source url + bytes copied/bytes total + datetime + error string + P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot + date-time-value + true + datetime + int + date-time-value + + + value + + + + blob-prefix + + + + + ''' + if response is None or response.body is None: + return None + + blob_list = _list() + list_element = ETree.fromstring(response.body) + + setattr(blob_list, 'next_marker', list_element.findtext('NextMarker')) + + blobs_element = list_element.find('Blobs') + blob_prefix_elements = blobs_element.findall('BlobPrefix') + if blob_prefix_elements is not None: + for blob_prefix_element in blob_prefix_elements: + blob_list.append(blob_prefix_element.findtext('Name')) + + for blob_element in blobs_element.findall('Blob'): + blob_list.append(blob_element.findtext('Name')) + + return blob_list + + def _convert_xml_to_block_list(response): ''' @@ -450,3 +521,36 @@ def _parse_account_information(response): account_info.account_kind = response.headers['x-ms-account-kind'] return account_info + + +def _convert_xml_to_user_delegation_key(response): + """ + + + Guid + Guid + String, formatted ISO Date + String, formatted ISO Date + b + String, rest api version used to create delegation key + Ovg+o0K/0/2V8upg7AwlyAPCriEcOSXKuBu2Gv/PU70Y7aWDW3C2ZRmw6kYWqPWBaM1GosLkcSZkgsobAlT+Sw== + + + Converts xml response to UserDelegationKey class. + """ + + if response is None or response.body is None: + return None + + delegation_key = UserDelegationKey() + + key_element = ETree.fromstring(response.body) + delegation_key.signed_oid = key_element.findtext('SignedOid') + delegation_key.signed_tid = key_element.findtext('SignedTid') + delegation_key.signed_start = key_element.findtext('SignedStart') + delegation_key.signed_expiry = key_element.findtext('SignedExpiry') + delegation_key.signed_service = key_element.findtext('SignedService') + delegation_key.signed_version = key_element.findtext('SignedVersion') + delegation_key.value = key_element.findtext('Value') + + return delegation_key diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py index f1e9b540b0bf..757b49067475 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py @@ -123,8 +123,8 @@ def _decrypt_blob(require_encryption, key_encryption_key, key_resolver, except: if require_encryption: raise ValueError(_ERROR_DATA_NOT_ENCRYPTED) - else: - return content + + return content if not (encryption_data.encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256): raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py index 100b40898561..611d73db5093 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py @@ -4,7 +4,7 @@ # license information. # -------------------------------------------------------------------------- from xml.sax.saxutils import escape as xml_escape - +from datetime import date try: from xml.etree import cElementTree as ETree except ImportError: @@ -13,6 +13,9 @@ _encode_base64, _str, ) +from ..common._serialization import ( + _to_utc_datetime, +) from ..common._error import ( _validate_not_none, _ERROR_START_END_NEEDED_FOR_MD5, @@ -46,7 +49,8 @@ def _get_path(container_name=None, blob_name=None): def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True, - end_range_required=True, check_content_md5=False, align_to_page=False): + end_range_required=True, check_content_md5=False, align_to_page=False, + range_header_name='x-ms-range'): # If end range is provided, start range must be provided if start_range_required or end_range is not None: _validate_not_none('start_range', start_range) @@ -63,9 +67,9 @@ def _validate_and_format_range_headers(request, start_range, end_range, start_ra # Format based on whether end_range is present request.headers = request.headers or {} if end_range is not None: - request.headers['x-ms-range'] = 'bytes={0}-{1}'.format(start_range, end_range) + request.headers[range_header_name] = 'bytes={0}-{1}'.format(start_range, end_range) elif start_range is not None: - request.headers['x-ms-range'] = "bytes={0}-".format(start_range) + request.headers[range_header_name] = "bytes={0}-".format(start_range) # Content MD5 can only be provided for a complete range less than 4MB in size if check_content_md5: @@ -116,3 +120,34 @@ def _convert_block_list_to_xml(block_id_list): # return xml value return output + + +def _convert_delegation_key_info_to_xml(start_time, expiry_time): + """ + + + String, formatted ISO Date + String, formatted ISO Date + + + Convert key info to xml to send. + """ + if start_time is None or expiry_time is None: + raise ValueError("delegation key start/end times are required") + + key_info_element = ETree.Element('KeyInfo') + ETree.SubElement(key_info_element, 'Start').text = \ + _to_utc_datetime(start_time) if isinstance(start_time, date) else start_time + ETree.SubElement(key_info_element, 'Expiry').text = \ + _to_utc_datetime(expiry_time) if isinstance(expiry_time, date) else expiry_time + + # Add xml declaration and serialize + try: + stream = BytesIO() + ETree.ElementTree(key_info_element).write(stream, xml_declaration=True, encoding='utf-8', method='xml') + finally: + output = stream.getvalue() + stream.close() + + # return xml value + return output diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py index 8369cb3727e9..266852c21468 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py @@ -33,6 +33,7 @@ ) from ._serialization import ( _get_path, + _validate_and_format_range_headers, ) from ._upload_chunking import ( _AppendBlobChunkUploader, @@ -112,7 +113,7 @@ def __init__(self, account_name=None, account_key=None, sas_token=None, is_emula :param token_credential: A token credential used to authenticate HTTPS requests. The token value should be updated before its expiration. - :type `~..common.TokenCredential` + :type `~azure.storage.common.TokenCredential` ''' self.blob_type = _BlobTypes.AppendBlob super(AppendBlobService, self).__init__( @@ -286,6 +287,125 @@ def append_block(self, container_name, blob_name, block, return self._perform_request(request, _parse_append_block) + def append_block_from_url(self, container_name, blob_name, copy_source_url, source_range_start=None, + source_range_end=None, source_content_md5=None, source_if_modified_since=None, + source_if_unmodified_since=None, source_if_match=None, + source_if_none_match=None, maxsize_condition=None, + appendpos_condition=None, lease_id=None, if_modified_since=None, + if_unmodified_since=None, if_match=None, + if_none_match=None, timeout=None): + """ + Creates a new block to be committed as part of a blob, where the contents are read from a source url. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob. + :param str copy_source_url: + The URL of the source data. It can point to any Azure Blob or File, that is either public or has a + shared access signature attached. + :param int source_range_start: + This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source. + :param int source_range_end: + This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source. + :param str source_content_md5: + If given, the service will calculate the MD5 hash of the block content and compare against this value. + :param datetime source_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the source resource has been modified since the specified time. + :param datetime source_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the source resource has not been modified since the specified date/time. + :param str source_if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the source resource's ETag matches the value specified. + :param str source_if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the source resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the source resource does not exist, and fail the + operation if it does exist. + :param int maxsize_condition: + Optional conditional header. The max length in bytes permitted for + the append blob. If the Append Block operation would cause the blob + to exceed that limit or if the blob size is already greater than the + value specified in this header, the request will fail with + MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed). + :param int appendpos_condition: + Optional conditional header, used only for the Append Block operation. + A number indicating the byte offset to compare. Append Block will + succeed only if the append position is equal to this number. If it + is not, the request will fail with the + AppendPositionConditionNotMet error + (HTTP status code 412 - Precondition Failed). + :param str lease_id: + Required if the blob has an active lease. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + """ + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('copy_source_url', copy_source_url) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'appendblock', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-copy-source': copy_source_url, + 'x-ms-source-content-md5': source_content_md5, + 'x-ms-source-if-Modified-Since': _datetime_to_utc_string(source_if_modified_since), + 'x-ms-source-if-Unmodified-Since': _datetime_to_utc_string(source_if_unmodified_since), + 'x-ms-source-if-Match': _to_str(source_if_match), + 'x-ms-source-if-None-Match': _to_str(source_if_none_match), + 'x-ms-blob-condition-maxsize': _to_str(maxsize_condition), + 'x-ms-blob-condition-appendpos': _to_str(appendpos_condition), + 'x-ms-lease-id': _to_str(lease_id), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + + _validate_and_format_range_headers(request, source_range_start, source_range_end, + start_range_required=False, + end_range_required=False, + range_header_name="x-ms-source-range") + + return self._perform_request(request, _parse_append_block) + # ----Convenience APIs---------------------------------------------- def append_blob_from_path( diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py index adb9127ca5f5..e7ea8c7e6c73 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/baseblobservice.py @@ -37,6 +37,7 @@ _validate_decryption_required, _validate_access_policies, _ERROR_PARALLEL_NOT_SEEKABLE, + _validate_user_delegation_key, ) from ..common._http import HTTPRequest from ..common._serialization import ( @@ -50,7 +51,6 @@ ListGenerator, _OperationContext, ) - from .sharedaccesssignature import ( BlobSharedAccessSignature, ) @@ -59,12 +59,14 @@ _convert_xml_to_containers, _parse_blob, _convert_xml_to_blob_list, + _convert_xml_to_blob_name_list, _parse_container, _parse_snapshot_blob, _parse_lease, _convert_xml_to_signed_identifiers_and_access, _parse_base_properties, _parse_account_information, + _convert_xml_to_user_delegation_key, ) from ._download_chunking import _download_blob_chunks from ._error import ( @@ -74,6 +76,7 @@ from ._serialization import ( _get_path, _validate_and_format_range_headers, + _convert_delegation_key_info_to_xml, ) from .models import ( BlobProperties, @@ -189,7 +192,7 @@ def __init__(self, account_name=None, account_key=None, sas_token=None, is_emula :param token_credential: A token credential used to authenticate HTTPS requests. The token value should be updated before its expiration. - :type `~..common.TokenCredential` + :type `~azure.storage.common.TokenCredential` ''' service_params = _ServiceParameters.get_service_parameters( 'blob', @@ -327,7 +330,7 @@ def generate_account_shared_access_signature(self, resource_types, permission, restricts the request to those IP addresses. :param str protocol: Specifies the protocol permitted for a request made. The default value - is https,http. See :class:`~..common.models.Protocol` for possible values. + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. :return: A Shared Access Signature (sas) token. :rtype: str ''' @@ -343,7 +346,7 @@ def generate_container_shared_access_signature(self, container_name, start=None, id=None, ip=None, protocol=None, cache_control=None, content_disposition=None, content_encoding=None, content_language=None, - content_type=None): + content_type=None, user_delegation_key=None): ''' Generates a shared access signature for the container. Use the returned signature with the sas_token parameter of any BlobService. @@ -384,7 +387,7 @@ def generate_container_shared_access_signature(self, container_name, restricts the request to those IP addresses. :param str protocol: Specifies the protocol permitted for a request made. The default value - is https,http. See :class:`~..common.models.Protocol` for possible values. + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. :param str cache_control: Response header value for Cache-Control when resource is accessed using this shared access signature. @@ -400,14 +403,24 @@ def generate_container_shared_access_signature(self, container_name, :param str content_type: Response header value for Content-Type when resource is accessed using this shared access signature. + :param ~azure.storage.blob.models.UserDelegationKey user_delegation_key: + Instead of an account key, the user could pass in a user delegation key. + A user delegation key can be obtained from the service by authenticating with an AAD identity; + this can be accomplished by calling get_user_delegation_key. + When present, the SAS is signed with the user delegation key instead. :return: A Shared Access Signature (sas) token. :rtype: str ''' _validate_not_none('container_name', container_name) _validate_not_none('self.account_name', self.account_name) - _validate_not_none('self.account_key', self.account_key) - sas = BlobSharedAccessSignature(self.account_name, self.account_key) + if user_delegation_key is not None: + _validate_user_delegation_key(user_delegation_key) + sas = BlobSharedAccessSignature(self.account_name, user_delegation_key=user_delegation_key) + else: + _validate_not_none('self.account_key', self.account_key) + sas = BlobSharedAccessSignature(self.account_name, account_key=self.account_key) + return sas.generate_container( container_name, permission, @@ -424,19 +437,22 @@ def generate_container_shared_access_signature(self, container_name, ) def generate_blob_shared_access_signature( - self, container_name, blob_name, permission=None, + self, container_name, blob_name, snapshot=None, permission=None, expiry=None, start=None, id=None, ip=None, protocol=None, cache_control=None, content_disposition=None, content_encoding=None, content_language=None, - content_type=None): + content_type=None, user_delegation_key=None): ''' - Generates a shared access signature for the blob. + Generates a shared access signature for the blob or one of its snapshots. Use the returned signature with the sas_token parameter of any BlobService. :param str container_name: Name of container. :param str blob_name: Name of blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to grant permission. :param BlobPermissions permission: The permissions associated with the shared access signature. The user is restricted to operations allowed by the permissions. @@ -470,7 +486,7 @@ def generate_blob_shared_access_signature( restricts the request to those IP addresses. :param str protocol: Specifies the protocol permitted for a request made. The default value - is https,http. See :class:`~..common.models.Protocol` for possible values. + is https,http. See :class:`~azure.storage.common.models.Protocol` for possible values. :param str cache_control: Response header value for Cache-Control when resource is accessed using this shared access signature. @@ -486,20 +502,31 @@ def generate_blob_shared_access_signature( :param str content_type: Response header value for Content-Type when resource is accessed using this shared access signature. + :param ~azure.storage.blob.models.UserDelegationKey user_delegation_key: + Instead of an account key, the user could pass in a user delegation key. + A user delegation key can be obtained from the service by authenticating with an AAD identity; + this can be accomplished by calling get_user_delegation_key. + When present, the SAS is signed with the user delegation key instead. :return: A Shared Access Signature (sas) token. :rtype: str ''' _validate_not_none('container_name', container_name) _validate_not_none('blob_name', blob_name) _validate_not_none('self.account_name', self.account_name) - _validate_not_none('self.account_key', self.account_key) - sas = BlobSharedAccessSignature(self.account_name, self.account_key) + if user_delegation_key is not None: + _validate_user_delegation_key(user_delegation_key) + sas = BlobSharedAccessSignature(self.account_name, user_delegation_key=user_delegation_key) + else: + _validate_not_none('self.account_key', self.account_key) + sas = BlobSharedAccessSignature(self.account_name, account_key=self.account_key) + return sas.generate_blob( - container_name, - blob_name, - permission, - expiry, + container_name=container_name, + blob_name=blob_name, + snapshot=snapshot, + permission=permission, + expiry=expiry, start=start, id=id, ip=ip, @@ -511,6 +538,33 @@ def generate_blob_shared_access_signature( content_type=content_type, ) + def get_user_delegation_key(self, key_start_time, key_expiry_time, timeout=None): + """ + Obtain a user delegation key for the purpose of signing SAS tokens. + A token credential must be present on the service object for this request to succeed. + + :param datetime key_start_time: + A DateTime value. Indicates when the key becomes valid. + :param datetime key_expiry_time: + A DateTime value. Indicates when the key stops being valid. + :param int timeout: + The timeout parameter is expressed in seconds. + :return: + """ + _validate_not_none('key_start_time', key_start_time) + _validate_not_none('key_end_time', key_expiry_time) + + request = HTTPRequest() + request.method = 'POST' + request.host_locations = self._get_host_locations(secondary=True) + request.query = { + 'restype': 'service', + 'comp': 'userdelegationkey', + 'timeout': _int_to_str(timeout), + } + request.body = _get_request_body(_convert_delegation_key_info_to_xml(key_start_time, key_expiry_time)) + return self._perform_request(request, _convert_xml_to_user_delegation_key) + def list_containers(self, prefix=None, num_results=None, include_metadata=False, marker=None, timeout=None): ''' @@ -784,7 +838,7 @@ def set_container_acl(self, container_name, signed_identifiers=None, A dictionary of access policies to associate with the container. The dictionary may contain up to 5 elements. An empty dictionary will clear the access policies set on the service. - :type signed_identifiers: dict(str, :class:`~..common.models.AccessPolicy`) + :type signed_identifiers: dict(str, :class:`~azure.storage.common.models.AccessPolicy`) :param ~azure.storage.blob.models.PublicAccess public_access: Possible values include: container, blob. :param str lease_id: @@ -1244,14 +1298,66 @@ def list_blobs(self, container_name, prefix=None, num_results=None, include=None args = (container_name,) kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results, 'include': include, 'delimiter': delimiter, 'timeout': timeout, - '_context': operation_context} + '_context': operation_context, + '_converter': _convert_xml_to_blob_list} + resp = self._list_blobs(*args, **kwargs) + + return ListGenerator(resp, self._list_blobs, args, kwargs) + + def list_blob_names(self, container_name, prefix=None, num_results=None, + include=None, delimiter=None, marker=None, + timeout=None): + ''' + Returns a generator to list the blob names under the specified container. + The generator will lazily follow the continuation tokens returned by + the service and stop when all blobs have been returned or num_results is reached. + + If num_results is specified and the account has more than that number of + blobs, the generator will have a populated next_marker field once it + finishes. This marker can be used to create a new generator if more + results are desired. + + :param str container_name: + Name of existing container. + :param str prefix: + Filters the results to return only blobs whose names + begin with the specified prefix. + :param int num_results: + Specifies the maximum number of blobs to return, + including all :class:`BlobPrefix` elements. If the request does not specify + num_results or specifies a value greater than 5,000, the server will + return up to 5,000 items. Setting num_results to a value less than + or equal to zero results in error response code 400 (Bad Request). + :param ~azure.storage.blob.models.Include include: + Specifies one or more additional datasets to include in the response. + :param str delimiter: + When the request includes this parameter, the operation + returns a :class:`~azure.storage.blob.models.BlobPrefix` element in the + result list that acts as a placeholder for all blobs whose names begin + with the same substring up to the appearance of the delimiter character. + The delimiter may be a single character or a string. + :param str marker: + An opaque continuation token. This value can be retrieved from the + next_marker field of a previous generator object if num_results was + specified and that generator has finished enumerating results. If + specified, this generator will begin returning results from the point + where the previous generator stopped. + :param int timeout: + The timeout parameter is expressed in seconds. + ''' + operation_context = _OperationContext(location_lock=True) + args = (container_name,) + kwargs = {'prefix': prefix, 'marker': marker, 'max_results': num_results, + 'include': include, 'delimiter': delimiter, 'timeout': timeout, + '_context': operation_context, + '_converter': _convert_xml_to_blob_name_list} resp = self._list_blobs(*args, **kwargs) return ListGenerator(resp, self._list_blobs, args, kwargs) def _list_blobs(self, container_name, prefix=None, marker=None, max_results=None, include=None, delimiter=None, timeout=None, - _context=None): + _context=None, _converter=None): ''' Returns the list of blobs under the specified container. @@ -1320,7 +1426,7 @@ def _list_blobs(self, container_name, prefix=None, marker=None, 'timeout': _int_to_str(timeout), } - return self._perform_request(request, _convert_xml_to_blob_list, operation_context=_context) + return self._perform_request(request, _converter, operation_context=_context) def get_blob_account_information(self, container_name=None, blob_name=None, timeout=None): """ @@ -1371,7 +1477,7 @@ def get_blob_service_stats(self, timeout=None): :param int timeout: The timeout parameter is expressed in seconds. :return: The blob service stats. - :rtype: :class:`~..common.models.ServiceStats` + :rtype: :class:`~azure.storage.common.models.ServiceStats` ''' request = HTTPRequest() request.method = 'GET' @@ -1396,22 +1502,22 @@ def set_blob_service_properties( :param logging: Groups the Azure Analytics Logging settings. :type logging: - :class:`~..common.models.Logging` + :class:`~azure.storage.common.models.Logging` :param hour_metrics: The hour metrics settings provide a summary of request statistics grouped by API in hourly aggregates for blobs. :type hour_metrics: - :class:`~..common.models.Metrics` + :class:`~azure.storage.common.models.Metrics` :param minute_metrics: The minute metrics settings provide request statistics for each minute for blobs. :type minute_metrics: - :class:`~..common.models.Metrics` + :class:`~azure.storage.common.models.Metrics` :param cors: You can include up to five CorsRule elements in the list. If an empty list is specified, all CORS rules will be deleted, and CORS will be disabled for the service. - :type cors: list(:class:`~..common.models.CorsRule`) + :type cors: list(:class:`~azure.storage.common.models.CorsRule`) :param str target_version: Indicates the default version to use for requests if an incoming request's version is not specified. @@ -1421,13 +1527,18 @@ def set_blob_service_properties( The delete retention policy specifies whether to retain deleted blobs. It also specifies the number of days and versions of blob to keep. :type delete_retention_policy: - :class:`~..common.models.DeleteRetentionPolicy` + :class:`~azure.storage.common.models.DeleteRetentionPolicy` :param static_website: Specifies whether the static website feature is enabled, and if yes, indicates the index document and 404 error document to use. :type static_website: - :class:`~..common.models.StaticWebsite` + :class:`~azure.storage.common.models.StaticWebsite` ''' + if all(parameter is None for parameter in [logging, hour_metrics, minute_metrics, cors, target_version, + delete_retention_policy, static_website]): + + raise ValueError("set_blob_service_properties should be called with at least one parameter") + request = HTTPRequest() request.method = 'PUT' request.host_locations = self._get_host_locations() @@ -1450,7 +1561,7 @@ def get_blob_service_properties(self, timeout=None): :param int timeout: The timeout parameter is expressed in seconds. - :return: The blob :class:`~..common.models.ServiceProperties` with an attached + :return: The blob :class:`~azure.storage.common.models.ServiceProperties` with an attached target_version property. ''' request = HTTPRequest() @@ -1977,11 +2088,11 @@ def get_blob_to_stream( if max_connections > 1: if sys.version_info >= (3,) and not stream.seekable(): raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) - else: - try: - stream.seek(stream.tell()) - except (NotImplementedError, AttributeError): - raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) + + try: + stream.seek(stream.tell()) + except (NotImplementedError, AttributeError): + raise ValueError(_ERROR_PARALLEL_NOT_SEEKABLE) # The service only provides transactional MD5s for chunks under 4MB. # If validate_content is on, get only self.MAX_CHUNK_GET_SIZE for the first @@ -3071,7 +3182,7 @@ def copy_blob(self, container_name, blob_name, copy_source, destination_if_none_match, destination_lease_id, source_lease_id, timeout, - False) + False, False) def _copy_blob(self, container_name, blob_name, copy_source, metadata=None, @@ -3085,12 +3196,16 @@ def _copy_blob(self, container_name, blob_name, copy_source, destination_if_none_match=None, destination_lease_id=None, source_lease_id=None, timeout=None, - incremental_copy=False): + incremental_copy=False, + requires_sync=None): ''' See copy_blob for more details. This helper method - allows for standard copies as well as incremental copies which are only supported for page blobs. + allows for standard copies as well as incremental copies which are only supported for page blobs and sync + copies which are only supported for block blobs. :param bool incremental_copy: - The timeout parameter is expressed in seconds. + Performs an incremental copy operation on a page blob instead of a standard copy operation. + :param bool requires_sync: + Enforces that the service will not return a response until the copy is complete. ''' _validate_not_none('container_name', container_name) _validate_not_none('blob_name', blob_name) @@ -3137,8 +3252,10 @@ def _copy_blob(self, container_name, blob_name, copy_source, 'If-None-Match': _to_str(destination_if_none_match), 'x-ms-lease-id': _to_str(destination_lease_id), 'x-ms-source-lease-id': _to_str(source_lease_id), - 'x-ms-access-tier': _to_str(premium_page_blob_tier) + 'x-ms-access-tier': _to_str(premium_page_blob_tier), + 'x-ms-requires-sync': _to_str(requires_sync) } + _add_metadata_headers(metadata, request) return self._perform_request(request, _parse_properties, [BlobProperties]).copy diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py index abd693974656..26900d3c6149 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/blockblobservice.py @@ -39,7 +39,6 @@ from ..common._serialization import ( _len_plus ) - from ._deserialization import ( _convert_xml_to_block_list, _parse_base_properties, @@ -51,6 +50,7 @@ from ._serialization import ( _convert_block_list_to_xml, _get_path, + _validate_and_format_range_headers, ) from ._upload_chunking import ( _BlockBlobChunkUploader, @@ -316,8 +316,9 @@ def get_block_list(self, container_name, blob_name, snapshot=None, return self._perform_request(request, _convert_xml_to_block_list) - def put_block_from_url(self, container_name, blob_name, copy_source_url, source_range_start, source_range_end, - block_id, source_content_md5=None, lease_id=None, timeout=None): + def put_block_from_url(self, container_name, blob_name, copy_source_url, block_id, + source_range_start=None, source_range_end=None, + source_content_md5=None, lease_id=None, timeout=None): """ Creates a new block to be committed as part of a blob. @@ -349,8 +350,6 @@ def put_block_from_url(self, container_name, blob_name, copy_source_url, source_ _validate_not_none('container_name', container_name) _validate_not_none('blob_name', blob_name) _validate_not_none('copy_source_url', copy_source_url) - _validate_not_none('source_range_start', source_range_start) - _validate_not_none('source_range_end', source_range_end) _validate_not_none('block_id', block_id) request = HTTPRequest() @@ -365,9 +364,16 @@ def put_block_from_url(self, container_name, blob_name, copy_source_url, source_ request.headers = { 'x-ms-lease-id': _to_str(lease_id), 'x-ms-copy-source': copy_source_url, - 'x-ms-source-range': 'bytes=' + _to_str(source_range_start) + '-' + _to_str(source_range_end), 'x-ms-source-content-md5': source_content_md5, } + _validate_and_format_range_headers( + request, + source_range_start, + source_range_end, + start_range_required=False, + end_range_required=False, + range_header_name="x-ms-source-range" + ) self._perform_request(request) @@ -881,6 +887,136 @@ def set_standard_blob_tier( self._perform_request(request) + def copy_blob(self, container_name, blob_name, copy_source, + metadata=None, source_if_modified_since=None, + source_if_unmodified_since=None, source_if_match=None, + source_if_none_match=None, destination_if_modified_since=None, + destination_if_unmodified_since=None, destination_if_match=None, + destination_if_none_match=None, destination_lease_id=None, + source_lease_id=None, timeout=None, requires_sync=None): + + ''' + Copies a blob. This operation returns a copy operation + properties object. The copy operation may be configured to either be an + asynchronous, best-effort operation, or a synchronous operation. + + The source must be a block blob if requires_sync is true. Any existing + destination blob will be overwritten. The destination blob cannot be + modified while a copy operation is in progress. + + When copying from a block blob, all committed blocks and their block IDs are + copied. Uncommitted blocks are not copied. At the end of the copy operation, + the destination blob will have the same committed block count as the source. + + You can call get_blob_properties on the destination blob to check the status + of the copy operation. The final blob will be committed when the copy completes. + + :param str container_name: + Name of the destination container. The container must exist. + :param str blob_name: + Name of the destination blob. If the destination blob exists, it will + be overwritten. Otherwise, it will be created. + :param str copy_source: + A URL of up to 2 KB in length that specifies an Azure file or blob. + The value should be URL-encoded as it would appear in a request URI. + If the source is in another account, the source must either be public + or must be authenticated via a shared access signature. If the source + is public, no authentication is required. + Examples: + https://myaccount.blob.core.windows.net/mycontainer/myblob + https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot= + https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken + :param metadata: + Name-value pairs associated with the blob as metadata. If no name-value + pairs are specified, the operation will copy the metadata from the + source blob or file to the destination blob. If one or more name-value + pairs are specified, the destination blob is created with the specified + metadata, and metadata is not copied from the source blob or file. + :type metadata: dict(str, str) + :param datetime source_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source + blob has been modified since the specified date/time. + :param datetime source_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only if the source blob + has not been modified since the specified date/time. + :param ETag source_if_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the source blob only if its ETag matches the value + specified. If the ETag values do not match, the Blob service returns + status code 412 (Precondition Failed). This header cannot be specified + if the source is an Azure File. + :param ETag source_if_none_match: + An ETag value, or the wildcard character (*). Specify this conditional + header to copy the blob only if its ETag does not match the value + specified. If the values are identical, the Blob service returns status + code 412 (Precondition Failed). This header cannot be specified if the + source is an Azure File. + :param datetime destination_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has been modified since the specified date/time. + If the destination blob has not been modified, the Blob service returns + status code 412 (Precondition Failed). + :param datetime destination_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this conditional header to copy the blob only + if the destination blob has not been modified since the specified + date/time. If the destination blob has been modified, the Blob service + returns status code 412 (Precondition Failed). + :param ETag destination_if_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + matches the ETag value for an existing destination blob. If the ETag for + the destination blob does not match the ETag specified for If-Match, the + Blob service returns status code 412 (Precondition Failed). + :param ETag destination_if_none_match: + An ETag value, or the wildcard character (*). Specify an ETag value for + this conditional header to copy the blob only if the specified ETag value + does not match the ETag value for the destination blob. Specify the wildcard + character (*) to perform the operation only if the destination blob does not + exist. If the specified condition isn't met, the Blob service returns status + code 412 (Precondition Failed). + :param str destination_lease_id: + The lease ID specified for this header must match the lease ID of the + destination blob. If the request does not include the lease ID or it is not + valid, the operation fails with status code 412 (Precondition Failed). + :param str source_lease_id: + Specify this to perform the Copy Blob operation only if + the lease ID given matches the active lease ID of the source blob. + :param int timeout: + The timeout parameter is expressed in seconds. + :param bool requires_sync: + Enforces that the service will not return a response until the copy is complete. + :return: Copy operation properties such as status, source, and ID. + :rtype: :class:`~azure.storage.blob.models.CopyProperties` + ''' + + return self._copy_blob(container_name, blob_name, copy_source, + metadata, + premium_page_blob_tier=None, + source_if_modified_since=source_if_modified_since, + source_if_unmodified_since=source_if_unmodified_since, + source_if_match=source_if_match, + source_if_none_match=source_if_none_match, + destination_if_modified_since=destination_if_modified_since, + destination_if_unmodified_since=destination_if_unmodified_since, + destination_if_match=destination_if_match, + destination_if_none_match=destination_if_none_match, + destination_lease_id=destination_lease_id, + source_lease_id=source_lease_id, timeout=timeout, + incremental_copy=False, + requires_sync=requires_sync) + # -----Helper methods------------------------------------ def _put_blob(self, container_name, blob_name, blob, content_settings=None, metadata=None, validate_content=False, lease_id=None, if_modified_since=None, diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py index e39067aa3ac1..225d0f9370e8 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/models.py @@ -3,7 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- - from ..common._common_conversion import _to_str @@ -670,13 +669,19 @@ class ContainerPermissions(object): a container SAS. Use an account SAS instead. ''' - def __init__(self, read=False, write=False, delete=False, list=False, + def __init__(self, read=False, add=False, create=False, write=False, delete=False, list=False, _str=None): ''' :param bool read: Read the content, properties, metadata or block list of any blob in the container. Use any blob in the container as the source of a copy operation. - :param bool write: + :param bool add: + Add a block to any append blob in the container. + :param bool create: + Write a new blob to the container, snapshot any blob in the container, or copy a blob to + a new blob in the container. Note: You cannot grant permissions to create a container + with a container SAS. Use an account SAS to create a container instead. + :param bool write: For any blob in the container, create or write content, properties, metadata, or block list. Snapshot or lease the blob. Resize the blob (page blob only). Use the blob as the destination of a copy operation @@ -694,6 +699,8 @@ def __init__(self, read=False, write=False, delete=False, list=False, if not _str: _str = '' self.read = read or ('r' in _str) + self.add = add or ('a' in _str) + self.create = create or ('c' in _str) self.write = write or ('w' in _str) self.delete = delete or ('d' in _str) self.list = list or ('l' in _str) @@ -706,6 +713,8 @@ def __add__(self, other): def __str__(self): return (('r' if self.read else '') + + ('a' if self.add else '') + + ('c' if self.create else '') + ('w' if self.write else '') + ('d' if self.delete else '') + ('l' if self.list else '')) @@ -715,6 +724,8 @@ def __str__(self): ContainerPermissions.LIST = ContainerPermissions(list=True) ContainerPermissions.READ = ContainerPermissions(read=True) ContainerPermissions.WRITE = ContainerPermissions(write=True) +ContainerPermissions.ADD = ContainerPermissions(add=True) +ContainerPermissions.CREATE = ContainerPermissions(create=True) class PremiumPageBlobTier(object): @@ -779,3 +790,36 @@ class AccountInformation(object): def __init__(self): self.sku_name = None self.account_kind = None + + +class UserDelegationKey(object): + """ + Represents a user delegation key, provided to the user by Azure Storage + based on their Azure Active Directory access token. + + The fields are saved as simple strings since the user does not have to interact with this object; + to generate an identify SAS, the user can simply pass it to the right API. + + :ivar str signed_oid: + Object ID of this token. + :ivar str signed_tid: + Tenant ID of the tenant that issued this token. + :ivar str signed_start: + The datetime this token becomes valid. + :ivar str signed_expiry: + The datetime this token expires. + :ivar str signed_service: + What service this key is valid for. + :ivar str signed_version: + The version identifier of the REST service that created this token. + :ivar str value: + The user delegation key. + """ + def __init__(self): + self.signed_oid = None + self.signed_tid = None + self.signed_start = None + self.signed_expiry = None + self.signed_service = None + self.signed_version = None + self.value = None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py index 476d55a49071..3c3217b285f3 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/pageblobservice.py @@ -6,7 +6,6 @@ import sys from os import path - from ..common._common_conversion import ( _int_to_str, _to_str, @@ -29,7 +28,6 @@ _get_data_bytes_only, _add_metadata_headers, ) - from ._deserialization import ( _convert_xml_to_page_ranges, _parse_page_properties, @@ -382,6 +380,136 @@ def update_page( timeout=timeout ) + def update_page_from_url(self, container_name, blob_name, start_range, end_range, copy_source_url, + source_range_start, source_content_md5=None, source_if_modified_since=None, + source_if_unmodified_since=None, source_if_match=None, source_if_none_match=None, + lease_id=None, if_sequence_number_lte=None, if_sequence_number_lt=None, + if_sequence_number_eq=None, if_modified_since=None, if_unmodified_since=None, + if_match=None, if_none_match=None, timeout=None): + """ + Updates a range of pages to a page blob where the contents are read from a URL. + + :param str container_name: + Name of existing container. + :param str blob_name: + Name of blob. + :param int start_range: + Start of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param int end_range: + End of byte range to use for writing to a section of the blob. + Pages must be aligned with 512-byte boundaries, the start offset + must be a modulus of 512 and the end offset must be a modulus of + 512-1. Examples of valid byte ranges are 0-511, 512-1023, etc. + :param str copy_source_url: + The URL of the source data. It can point to any Azure Blob or File, that is either public or has a + shared access signature attached. + :param int source_range_start: + This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source. + The service will read the same number of bytes as the destination range (end_range-start_range). + :param str source_content_md5: + If given, the service will calculate the MD5 hash of the block content and compare against this value. + :param datetime source_if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the source resource has been modified since the specified time. + :param datetime source_if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the source resource has not been modified since the specified date/time. + :param str source_if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the source resource's ETag matches the value specified. + :param str source_if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the source resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the source resource does not exist, and fail the + operation if it does exist. + :param str lease_id: + Required if the blob has an active lease. + :param int if_sequence_number_lte: + If the blob's sequence number is less than or equal to + the specified value, the request proceeds; otherwise it fails. + :param int if_sequence_number_lt: + If the blob's sequence number is less than the specified + value, the request proceeds; otherwise it fails. + :param int if_sequence_number_eq: + If the blob's sequence number is equal to the specified + value, the request proceeds; otherwise it fails. + :param datetime if_modified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only + if the resource has been modified since the specified time. + :param datetime if_unmodified_since: + A DateTime value. Azure expects the date value passed in to be UTC. + If timezone is included, any non-UTC datetimes will be converted to UTC. + If a date is passed in without timezone info, it is assumed to be UTC. + Specify this header to perform the operation only if + the resource has not been modified since the specified date/time. + :param str if_match: + An ETag value, or the wildcard character (*). Specify this header to perform + the operation only if the resource's ETag matches the value specified. + :param str if_none_match: + An ETag value, or the wildcard character (*). Specify this header + to perform the operation only if the resource's ETag does not match + the value specified. Specify the wildcard character (*) to perform + the operation only if the resource does not exist, and fail the + operation if it does exist. + :param int timeout: + The timeout parameter is expressed in seconds. + """ + _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key) + _validate_not_none('container_name', container_name) + _validate_not_none('blob_name', blob_name) + _validate_not_none('copy_source_url', copy_source_url) + + request = HTTPRequest() + request.method = 'PUT' + request.host_locations = self._get_host_locations() + request.path = _get_path(container_name, blob_name) + request.query = { + 'comp': 'page', + 'timeout': _int_to_str(timeout), + } + request.headers = { + 'x-ms-page-write': 'update', + 'x-ms-copy-source': copy_source_url, + 'x-ms-source-content-md5': source_content_md5, + 'x-ms-source-if-Modified-Since': _datetime_to_utc_string(source_if_modified_since), + 'x-ms-source-if-Unmodified-Since': _datetime_to_utc_string(source_if_unmodified_since), + 'x-ms-source-if-Match': _to_str(source_if_match), + 'x-ms-source-if-None-Match': _to_str(source_if_none_match), + 'x-ms-lease-id': _to_str(lease_id), + 'x-ms-if-sequence-number-le': _to_str(if_sequence_number_lte), + 'x-ms-if-sequence-number-lt': _to_str(if_sequence_number_lt), + 'x-ms-if-sequence-number-eq': _to_str(if_sequence_number_eq), + 'If-Modified-Since': _datetime_to_utc_string(if_modified_since), + 'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since), + 'If-Match': _to_str(if_match), + 'If-None-Match': _to_str(if_none_match) + } + _validate_and_format_range_headers( + request, + start_range, + end_range, + align_to_page=True) + _validate_and_format_range_headers( + request, + source_range_start, + source_range_start+(end_range-start_range), + range_header_name="x-ms-source-range") + + return self._perform_request(request, _parse_page_properties) + def clear_page( self, container_name, blob_name, start_range, end_range, lease_id=None, if_sequence_number_lte=None, diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py index 6947e7e10ff7..a9538c9d65ba 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/sharedaccesssignature.py @@ -7,9 +7,13 @@ from ..common.sharedaccesssignature import ( SharedAccessSignature, _SharedAccessHelper, + _QueryStringConstants, + _sign_string, ) - from ._constants import X_MS_VERSION +from ..common._serialization import ( + url_quote, +) class BlobSharedAccessSignature(SharedAccessSignature): @@ -20,28 +24,36 @@ class BlobSharedAccessSignature(SharedAccessSignature): generate_*_shared_access_signature method directly. ''' - def __init__(self, account_name, account_key): + def __init__(self, account_name, account_key=None, user_delegation_key=None): ''' :param str account_name: The storage account name used to generate the shared access signatures. :param str account_key: The access key to generate the shares access signatures. + :param ~azure.storage.blob.models.UserDelegationKey user_delegation_key: + Instead of an account key, the user could pass in a user delegation key. + A user delegation key can be obtained from the service by authenticating with an AAD identity; + this can be accomplished by calling get_user_delegation_key on any Blob service object. ''' super(BlobSharedAccessSignature, self).__init__(account_name, account_key, x_ms_version=X_MS_VERSION) + self.user_delegation_key = user_delegation_key - def generate_blob(self, container_name, blob_name, permission=None, + def generate_blob(self, container_name, blob_name, snapshot=None, permission=None, expiry=None, start=None, id=None, ip=None, protocol=None, cache_control=None, content_disposition=None, content_encoding=None, content_language=None, content_type=None): ''' - Generates a shared access signature for the blob. + Generates a shared access signature for the blob or one of its snapshots. Use the returned signature with the sas_token parameter of any BlobService. :param str container_name: Name of container. :param str blob_name: Name of blob. + :param str snapshot: + The snapshot parameter is an opaque DateTime value that, + when present, specifies the blob snapshot to grant permission. :param BlobPermissions permission: The permissions associated with the shared access signature. The user is restricted to operations allowed by the permissions. @@ -95,14 +107,16 @@ def generate_blob(self, container_name, blob_name, permission=None, ''' resource_path = container_name + '/' + blob_name - sas = _SharedAccessHelper() + sas = _BlobSharedAccessHelper() sas.add_base(permission, expiry, start, ip, protocol, self.x_ms_version) sas.add_id(id) - sas.add_resource('b') + sas.add_resource('b' if snapshot is None else 'bs') + sas.add_timestamp(snapshot) sas.add_override_response_headers(cache_control, content_disposition, content_encoding, content_language, content_type) - sas.add_resource_signature(self.account_name, self.account_key, 'blob', resource_path) + sas.add_resource_signature(self.account_name, self.account_key, resource_path, + user_delegation_key=self.user_delegation_key) return sas.get_token() @@ -168,13 +182,94 @@ def generate_container(self, container_name, permission=None, expiry=None, Response header value for Content-Type when resource is accessed using this shared access signature. ''' - sas = _SharedAccessHelper() + sas = _BlobSharedAccessHelper() sas.add_base(permission, expiry, start, ip, protocol, self.x_ms_version) sas.add_id(id) sas.add_resource('c') sas.add_override_response_headers(cache_control, content_disposition, content_encoding, content_language, content_type) - sas.add_resource_signature(self.account_name, self.account_key, 'blob', container_name) - + sas.add_resource_signature(self.account_name, self.account_key, container_name, + user_delegation_key=self.user_delegation_key) return sas.get_token() + + +class _BlobQueryStringConstants(_QueryStringConstants): + SIGNED_TIMESTAMP = 'snapshot' + SIGNED_OID = 'skoid' + SIGNED_TID = 'sktid' + SIGNED_KEY_START = 'skt' + SIGNED_KEY_EXPIRY = 'ske' + SIGNED_KEY_SERVICE = 'sks' + SIGNED_KEY_VERSION = 'skv' + + +class _BlobSharedAccessHelper(_SharedAccessHelper): + def __init__(self): + super(_BlobSharedAccessHelper, self).__init__() + + def add_timestamp(self, timestamp): + self._add_query(_BlobQueryStringConstants.SIGNED_TIMESTAMP, timestamp) + + def get_value_to_append(self, query): + return_value = self.query_dict.get(query) or '' + return return_value + '\n' + + def add_resource_signature(self, account_name, account_key, path, user_delegation_key=None): + if path[0] != '/': + path = '/' + path + + canonicalized_resource = '/blob/' + account_name + path + '\n' + + # Form the string to sign from shared_access_policy and canonicalized + # resource. The order of values is important. + string_to_sign = \ + (self.get_value_to_append(_BlobQueryStringConstants.SIGNED_PERMISSION) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_START) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_EXPIRY) + + canonicalized_resource) + + if user_delegation_key is not None: + self._add_query(_BlobQueryStringConstants.SIGNED_OID, user_delegation_key.signed_oid) + self._add_query(_BlobQueryStringConstants.SIGNED_TID, user_delegation_key.signed_tid) + self._add_query(_BlobQueryStringConstants.SIGNED_KEY_START, user_delegation_key.signed_start) + self._add_query(_BlobQueryStringConstants.SIGNED_KEY_EXPIRY, user_delegation_key.signed_expiry) + self._add_query(_BlobQueryStringConstants.SIGNED_KEY_SERVICE, user_delegation_key.signed_service) + self._add_query(_BlobQueryStringConstants.SIGNED_KEY_VERSION, user_delegation_key.signed_version) + + string_to_sign += \ + (self.get_value_to_append(_BlobQueryStringConstants.SIGNED_OID) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_TID) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_KEY_START) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_KEY_EXPIRY) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_KEY_SERVICE) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_KEY_VERSION)) + else: + string_to_sign += self.get_value_to_append(_BlobQueryStringConstants.SIGNED_IDENTIFIER) + + string_to_sign += \ + (self.get_value_to_append(_BlobQueryStringConstants.SIGNED_IP) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_PROTOCOL) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_VERSION) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_RESOURCE) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_TIMESTAMP) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_CACHE_CONTROL) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_CONTENT_DISPOSITION) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_CONTENT_ENCODING) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_CONTENT_LANGUAGE) + + self.get_value_to_append(_BlobQueryStringConstants.SIGNED_CONTENT_TYPE)) + + # remove the trailing newline + if string_to_sign[-1] == '\n': + string_to_sign = string_to_sign[:-1] + + self._add_query(_BlobQueryStringConstants.SIGNED_SIGNATURE, + _sign_string(account_key if user_delegation_key is None else user_delegation_key.value, + string_to_sign)) + + def get_token(self): + # a conscious decision was made to exclude the timestamp in the generated token + # this is to avoid having two snapshot ids in the query parameters when the user appends the snapshot timestamp + exclude = [_BlobQueryStringConstants.SIGNED_TIMESTAMP] + return '&'.join(['{0}={1}'.format(n, url_quote(v)) + for n, v in self.query_dict.items() if v is not None and n not in exclude]) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py index 797c97069ee1..a646e3811588 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/__init__.py @@ -36,3 +36,4 @@ SharedAccessSignature, ) from .tokencredential import TokenCredential +from ._error import AzureSigningError diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py index 15c15b9ea560..13940f97b6f7 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_auth.py @@ -10,10 +10,21 @@ DEV_ACCOUNT_NAME, DEV_ACCOUNT_SECONDARY_NAME ) +import sys +if sys.version_info >= (3,): + from urllib.parse import parse_qsl +else: + from urlparse import parse_qsl + import logging logger = logging.getLogger(__name__) +from ._error import ( + AzureSigningError, + _wrap_exception, +) + class _StorageSharedKeyAuthentication(object): def __init__(self, account_name, account_key, is_emulated=False): @@ -54,9 +65,14 @@ def _get_canonicalized_headers(self, request): return string_to_sign def _add_authorization_header(self, request, string_to_sign): - signature = _sign_string(self.account_key, string_to_sign) - auth_string = 'SharedKey ' + self.account_name + ':' + signature - request.headers['Authorization'] = auth_string + try: + signature = _sign_string(self.account_key, string_to_sign) + auth_string = 'SharedKey ' + self.account_name + ':' + signature + request.headers['Authorization'] = auth_string + except Exception as ex: + # Wrap any error that occurred as signing error + # Doing so will clarify/locate the source of problem + raise _wrap_exception(ex, AzureSigningError) class _StorageSharedKeyAuthentication(_StorageSharedKeyAuthentication): @@ -100,18 +116,14 @@ def __init__(self, sas_token): # ignore ?-prefix (added by tools such as Azure Portal) on sas tokens # doing so avoids double question marks when signing if sas_token[0] == '?': - self.sas_token = sas_token[1:] - else: - self.sas_token = sas_token + sas_token = sas_token[1:] + + self.sas_qs = parse_qsl(sas_token) def sign_request(self, request): - # if 'sig=' is present, then the request has already been signed + # if 'sig' is present, then the request has already been signed # as is the case when performing retries - if 'sig=' in request.path: + if 'sig' in request.query: return - if '?' in request.path: - request.path += '&' - else: - request.path += '?' - request.path += self.sas_token + request.query.update(self.sas_qs) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py index 1388fddeb625..6836cf91fad6 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_connection.py @@ -60,9 +60,10 @@ def __init__(self, service, account_name=None, account_key=None, sas_token=None, # Only set the account key if a sas_token is not present to allow sas to be used with the emulator self.account_key = DEV_ACCOUNT_KEY if not self.sas_token else None + emulator_endpoint = _EMULATOR_ENDPOINTS[service] if custom_domain is None else custom_domain - self.primary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_NAME) - self.secondary_endpoint = '{}/{}'.format(_EMULATOR_ENDPOINTS[service], DEV_ACCOUNT_SECONDARY_NAME) + self.primary_endpoint = '{}/{}'.format(emulator_endpoint, DEV_ACCOUNT_NAME) + self.secondary_endpoint = '{}/{}'.format(emulator_endpoint, DEV_ACCOUNT_SECONDARY_NAME) else: # Strip whitespace from the key if self.account_key: @@ -108,7 +109,7 @@ def get_service_parameters(service, account_name=None, account_key=None, sas_tok if connection_string: params = _ServiceParameters._from_connection_string(connection_string, service) elif is_emulated: - params = _ServiceParameters(service, is_emulated=True) + params = _ServiceParameters(service, is_emulated=True, custom_domain=custom_domain) elif account_name: if protocol.lower() != 'https' and token_credential is not None: raise ValueError("Token credential is only supported with HTTPS.") diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py index 22516d640757..22d93b3a2cd6 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_constants.py @@ -7,7 +7,7 @@ import sys __author__ = 'Microsoft Corp. ' -__version__ = '1.3.0' +__version__ = '2.0.0' # UserAgent string sample: 'Azure-Storage/0.37.0-0.38.0 (Python CPython 3.4.2; Windows 8)' # First version(0.37.0) is the common package, and the second version(0.38.0) is the service package @@ -45,3 +45,7 @@ # Encryption constants _ENCRYPTION_PROTOCOL_V1 = '1.0' + +_AUTHORIZATION_HEADER_NAME = 'Authorization' +_COPY_SOURCE_HEADER_NAME = 'x-ms-copy-source' +_REDACTED_VALUE = 'REDACTED' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py index 90faa0124ab2..5c8e393197c9 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/_error.py @@ -181,3 +181,38 @@ def _validate_kek_id(kid, resolved_id): def _validate_encryption_unsupported(require_encryption, key_encryption_key): if require_encryption or (key_encryption_key is not None): raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) + + +def _validate_user_delegation_key(user_delegation_key): + _validate_not_none('user_delegation_key.signed_oid', user_delegation_key.signed_oid) + _validate_not_none('user_delegation_key.signed_tid', user_delegation_key.signed_tid) + _validate_not_none('user_delegation_key.signed_start', user_delegation_key.signed_start) + _validate_not_none('user_delegation_key.signed_expiry', user_delegation_key.signed_expiry) + _validate_not_none('user_delegation_key.signed_version', user_delegation_key.signed_version) + _validate_not_none('user_delegation_key.signed_service', user_delegation_key.signed_service) + _validate_not_none('user_delegation_key.value', user_delegation_key.value) + + +# wraps a given exception with the desired exception type +def _wrap_exception(ex, desired_type): + msg = "" + if len(ex.args) > 0: + msg = ex.args[0] + if version_info >= (3,): + # Automatic chaining in Python 3 means we keep the trace + return desired_type(msg) + else: + # There isn't a good solution in 2 for keeping the stack trace + # in general, or that will not result in an error in 3 + # However, we can keep the previous error type and message + # TODO: In the future we will log the trace + return desired_type('{}: {}'.format(ex.__class__.__name__, msg)) + + +class AzureSigningError(AzureException): + """ + Represents a fatal error when attempting to sign a request. + In general, the cause of this exception is user error. For example, the given account key is not valid. + Please visit https://docs.microsoft.com/en-us/azure/storage/common/storage-create-storage-account for more info. + """ + pass diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py index f3ac1aa7be70..459146849163 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/cloudstorageaccount.py @@ -19,18 +19,6 @@ SharedAccessSignature, ) -''' -from azure.storage.common._error import _validate_not_none -from azure.storage.common.models import ( - ResourceTypes, - Services, - AccountPermissions, -) -from azure.storage.common.sharedaccesssignature import ( - SharedAccessSignature, -) -''' - class CloudStorageAccount(object): """ @@ -39,7 +27,8 @@ class CloudStorageAccount(object): use the factory or can construct the appropriate service directly. """ - def __init__(self, account_name=None, account_key=None, sas_token=None, is_emulated=None): + def __init__(self, account_name=None, account_key=None, sas_token=None, + is_emulated=None, endpoint_suffix=None): ''' :param str account_name: The storage account name. This is used to authenticate requests @@ -52,13 +41,17 @@ def __init__(self, account_name=None, account_key=None, sas_token=None, is_emula instead of the account key. If account key and sas token are both specified, account key will be used to sign. :param bool is_emulated: - Whether to use the emulator. Defaults to False. If specified, will + Whether to use the emulator. Defaults to False. If specified, will override all other parameters. + :param str endpoint_suffix: + The host base component of the url, minus the account name. Defaults + to Azure (core.windows.net). Override this to use a sovereign cloud. ''' self.account_name = account_name self.account_key = account_key self.sas_token = sas_token self.is_emulated = is_emulated + self.endpoint_suffix = endpoint_suffix def create_block_blob_service(self): ''' @@ -72,7 +65,8 @@ def create_block_blob_service(self): from azure.storage.blob.blockblobservice import BlockBlobService return BlockBlobService(self.account_name, self.account_key, sas_token=self.sas_token, - is_emulated=self.is_emulated) + is_emulated=self.is_emulated, + endpoint_suffix=self.endpoint_suffix) except ImportError: raise Exception('The package azure-storage-blob is required. ' + 'Please install it using "pip install azure-storage-blob"') @@ -89,7 +83,8 @@ def create_page_blob_service(self): from azure.storage.blob.pageblobservice import PageBlobService return PageBlobService(self.account_name, self.account_key, sas_token=self.sas_token, - is_emulated=self.is_emulated) + is_emulated=self.is_emulated, + endpoint_suffix=self.endpoint_suffix) except ImportError: raise Exception('The package azure-storage-blob is required. ' + 'Please install it using "pip install azure-storage-blob"') @@ -106,7 +101,8 @@ def create_append_blob_service(self): from azure.storage.blob.appendblobservice import AppendBlobService return AppendBlobService(self.account_name, self.account_key, sas_token=self.sas_token, - is_emulated=self.is_emulated) + is_emulated=self.is_emulated, + endpoint_suffix=self.endpoint_suffix) except ImportError: raise Exception('The package azure-storage-blob is required. ' + 'Please install it using "pip install azure-storage-blob"') @@ -123,7 +119,8 @@ def create_queue_service(self): from azure.storage.queue.queueservice import QueueService return QueueService(self.account_name, self.account_key, sas_token=self.sas_token, - is_emulated=self.is_emulated) + is_emulated=self.is_emulated, + endpoint_suffix=self.endpoint_suffix) except ImportError: raise Exception('The package azure-storage-queue is required. ' + 'Please install it using "pip install azure-storage-queue"') @@ -139,7 +136,8 @@ def create_file_service(self): try: from azure.storage.file.fileservice import FileService return FileService(self.account_name, self.account_key, - sas_token=self.sas_token) + sas_token=self.sas_token, + endpoint_suffix=self.endpoint_suffix) except ImportError: raise Exception('The package azure-storage-file is required. ' + 'Please install it using "pip install azure-storage-file"') diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py index 85764430259d..d18c84d80b55 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/retry.py @@ -151,7 +151,7 @@ def _retry(self, context, backoff): self._set_next_host_location(context) # rewind the request body if it is a stream - if hasattr(context.request.body, 'read'): + if hasattr(context.request, 'body') and hasattr(context.request.body, 'read'): # no position was saved, then retry would not work if context.body_position is None: return None diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py index c23201a85bcf..ae55b2a6aaed 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/sharedaccesssignature.py @@ -157,43 +157,6 @@ def add_override_response_headers(self, cache_control, self._add_query(_QueryStringConstants.SIGNED_CONTENT_LANGUAGE, content_language) self._add_query(_QueryStringConstants.SIGNED_CONTENT_TYPE, content_type) - def add_resource_signature(self, account_name, account_key, service, path): - def get_value_to_append(query): - return_value = self.query_dict.get(query) or '' - return return_value + '\n' - - if path[0] != '/': - path = '/' + path - - canonicalized_resource = '/' + service + '/' + account_name + path + '\n' - - # Form the string to sign from shared_access_policy and canonicalized - # resource. The order of values is important. - string_to_sign = \ - (get_value_to_append(_QueryStringConstants.SIGNED_PERMISSION) + - get_value_to_append(_QueryStringConstants.SIGNED_START) + - get_value_to_append(_QueryStringConstants.SIGNED_EXPIRY) + - canonicalized_resource + - get_value_to_append(_QueryStringConstants.SIGNED_IDENTIFIER) + - get_value_to_append(_QueryStringConstants.SIGNED_IP) + - get_value_to_append(_QueryStringConstants.SIGNED_PROTOCOL) + - get_value_to_append(_QueryStringConstants.SIGNED_VERSION)) - - if service == 'blob' or service == 'file': - string_to_sign += \ - (get_value_to_append(_QueryStringConstants.SIGNED_CACHE_CONTROL) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_DISPOSITION) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_ENCODING) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_LANGUAGE) + - get_value_to_append(_QueryStringConstants.SIGNED_CONTENT_TYPE)) - - # remove the trailing newline - if string_to_sign[-1] == '\n': - string_to_sign = string_to_sign[:-1] - - self._add_query(_QueryStringConstants.SIGNED_SIGNATURE, - _sign_string(account_key, string_to_sign)) - def add_account_signature(self, account_name, account_key): def get_value_to_append(query): return_value = self.query_dict.get(query) or '' diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py index 859a729df466..41ae2c627805 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/common/storageclient.py @@ -4,14 +4,12 @@ # license information. # -------------------------------------------------------------------------- -import sys +import requests from abc import ABCMeta import logging - -logger = logging.getLogger(__name__) from time import sleep +import sys -import requests from azure.common import ( AzureException, AzureHttpError, @@ -23,10 +21,15 @@ DEFAULT_USER_AGENT_STRING, USER_AGENT_STRING_PREFIX, USER_AGENT_STRING_SUFFIX, + _AUTHORIZATION_HEADER_NAME, + _REDACTED_VALUE, + _COPY_SOURCE_HEADER_NAME, ) from ._error import ( _ERROR_DECRYPTION_FAILURE, _http_error_handler, + _wrap_exception, + AzureSigningError, ) from ._http import HTTPError from ._http.httpclient import _HTTPClient @@ -41,6 +44,23 @@ ) from .retry import ExponentialRetry from io import UnsupportedOperation +from .sharedaccesssignature import _QueryStringConstants + +if sys.version_info >= (3,): + from urllib.parse import ( + urlparse, + parse_qsl, + urlunparse, + urlencode, + ) +else: + from urlparse import ( + urlparse, + parse_qsl, + urlunparse, + ) + from urllib import urlencode +logger = logging.getLogger(__name__) class StorageClient(object): @@ -210,6 +230,36 @@ def extract_date_and_request_id(retry_context): else: return "" + @staticmethod + def _scrub_headers(headers): + # make a copy to avoid contaminating the request + clean_headers = headers.copy() + + if _AUTHORIZATION_HEADER_NAME in clean_headers: + clean_headers[_AUTHORIZATION_HEADER_NAME] = _REDACTED_VALUE + + # in case of copy operations, there could be a SAS signature present in the header value + if _COPY_SOURCE_HEADER_NAME in clean_headers \ + and _QueryStringConstants.SIGNED_SIGNATURE + "=" in clean_headers[_COPY_SOURCE_HEADER_NAME]: + # take the url apart and scrub away the signed signature + scheme, netloc, path, params, query, fragment = urlparse(clean_headers[_COPY_SOURCE_HEADER_NAME]) + parsed_qs = dict(parse_qsl(query)) + parsed_qs[_QueryStringConstants.SIGNED_SIGNATURE] = _REDACTED_VALUE + + # the SAS needs to be put back together + clean_headers[_COPY_SOURCE_HEADER_NAME] = urlunparse( + (scheme, netloc, path, params, urlencode(parsed_qs), fragment)) + return clean_headers + + @staticmethod + def _scrub_query_parameters(query): + # make a copy to avoid contaminating the request + clean_queries = query.copy() + + if _QueryStringConstants.SIGNED_SIGNATURE in clean_queries: + clean_queries[_QueryStringConstants.SIGNED_SIGNATURE] = _REDACTED_VALUE + return clean_queries + def _perform_request(self, request, parser=None, parser_args=None, operation_context=None, expected_errors=None): ''' Sends the request and return response. Catches HTTPError and hands it @@ -258,12 +308,14 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con retry_context.request = request # Log the request before it goes out - logger.info("%s Outgoing request: Method=%s, Path=%s, Query=%s, Headers=%s.", - client_request_id_prefix, - request.method, - request.path, - request.query, - str(request.headers).replace('\n', '')) + # Avoid unnecessary scrubbing if the logger is not on + if logger.isEnabledFor(logging.INFO): + logger.info("%s Outgoing request: Method=%s, Path=%s, Query=%s, Headers=%s.", + client_request_id_prefix, + request.method, + request.path, + self._scrub_query_parameters(request.query), + str(self._scrub_headers(request.headers)).replace('\n', '')) # Perform the request response = self._httpclient.perform_request(request) @@ -306,21 +358,13 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con raise ex except Exception as ex: retry_context.exception = ex - if sys.version_info >= (3,): - # Automatic chaining in Python 3 means we keep the trace - raise AzureException(ex.args[0]) - else: - # There isn't a good solution in 2 for keeping the stack trace - # in general, or that will not result in an error in 3 - # However, we can keep the previous error type and message - # TODO: In the future we will log the trace - msg = "" - if len(ex.args) > 0: - msg = ex.args[0] - raise AzureException('{}: {}'.format(ex.__class__.__name__, msg)) + raise _wrap_exception(ex, AzureException) except AzureException as ex: # only parse the strings used for logging if logging is at least enabled for CRITICAL + exception_str_in_one_line = '' + status_code = '' + timestamp_and_request_id = '' if logger.isEnabledFor(logging.CRITICAL): exception_str_in_one_line = str(ex).replace('\n', '') status_code = retry_context.response.status if retry_context.response is not None else 'Unknown' @@ -335,6 +379,11 @@ def _perform_request(self, request, parser=None, parser_args=None, operation_con status_code, exception_str_in_one_line) raise ex + elif isinstance(ex, AzureSigningError): + logger.info("%s Unable to sign the request: Exception=%s.", + client_request_id_prefix, + exception_str_in_one_line) + raise ex logger.info("%s Operation failed: checking if the operation should be retried. " "Current retry count=%s, %s, HTTP status code=%s, Exception=%s.", diff --git a/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/vendor_azure_storage_version.md b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/vendor_azure_storage_version.md new file mode 100644 index 000000000000..caabbbca7d46 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/vendor_azure_storage_version.md @@ -0,0 +1,2 @@ +# azure-storage-blob 2.0.1 +# azure-storage-common 2.0.0