Vendor storage 2.0.1 within EPH

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/__init__.py

@@ -1 +1 @@
-__import__('pkg_resources').declare_namespace(__name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/__init__.py

@@ -1 +1 @@
-__import__('pkg_resources').declare_namespace(__name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_constants.py

@@ -5,10 +5,10 @@
 # --------------------------------------------------------------------------
 
 __author__ = 'Microsoft Corp. <ptvshelp@microsoft.com>'
-__version__ = '1.3.1'
+__version__ = '2.0.1'
 
 # x-ms-version for storage service.
-X_MS_VERSION = '2018-03-28'
+X_MS_VERSION = '2018-11-09'
 
 # internal configurations, should not be changed
 _LARGE_BLOB_UPLOAD_MAX_READ_BUFFER_SIZE = 4 * 1024 * 1024

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_deserialization.py

@@ -10,7 +10,6 @@
     from xml.etree import cElementTree as ETree
 except ImportError:
     from xml.etree import ElementTree as ETree
-
 from ..common._common_conversion import (
     _decode_base64_to_text,
     _to_str,
@@ -37,6 +36,7 @@
     ResourceProperties,
     BlobPrefix,
     AccountInformation,
+    UserDelegationKey,
 )
 from ._encryption import _decrypt_blob
 from ..common.models import _list
@@ -352,6 +352,77 @@ def _convert_xml_to_blob_list(response):
     return blob_list
 
 
+def _convert_xml_to_blob_name_list(response):
+    '''
+    <?xml version="1.0" encoding="utf-8"?>
+    <EnumerationResults ServiceEndpoint="http://myaccount.blob.core.windows.net/" ContainerName="mycontainer">
+      <Prefix>string-value</Prefix>
+      <Marker>string-value</Marker>
+      <MaxResults>int-value</MaxResults>
+      <Delimiter>string-value</Delimiter>
+      <Blobs>
+        <Blob>
+          <Name>blob-name</name>
+          <Deleted>true</Deleted>
+          <Snapshot>date-time-value</Snapshot>
+          <Properties>
+            <Last-Modified>date-time-value</Last-Modified>
+            <Etag>etag</Etag>
+            <Content-Length>size-in-bytes</Content-Length>
+            <Content-Type>blob-content-type</Content-Type>
+            <Content-Encoding />
+            <Content-Language />
+            <Content-MD5 />
+            <Cache-Control />
+            <x-ms-blob-sequence-number>sequence-number</x-ms-blob-sequence-number>
+            <BlobType>BlockBlob|PageBlob|AppendBlob</BlobType>
+            <LeaseStatus>locked|unlocked</LeaseStatus>
+            <LeaseState>available | leased | expired | breaking | broken</LeaseState>
+            <LeaseDuration>infinite | fixed</LeaseDuration>
+            <CopyId>id</CopyId>
+            <CopyStatus>pending | success | aborted | failed </CopyStatus>
+            <CopySource>source url</CopySource>
+            <CopyProgress>bytes copied/bytes total</CopyProgress>
+            <CopyCompletionTime>datetime</CopyCompletionTime>
+            <CopyStatusDescription>error string</CopyStatusDescription>
+            <AccessTier>P4 | P6 | P10 | P20 | P30 | P40 | P50 | P60 | Archive | Cool | Hot</AccessTier>
+            <AccessTierChangeTime>date-time-value</AccessTierChangeTime>
+            <AccessTierInferred>true</AccessTierInferred>
+            <DeletedTime>datetime</DeletedTime>
+            <RemainingRetentionDays>int</RemainingRetentionDays>
+            <Creation-Time>date-time-value</Creation-Time>
+          </Properties>
+          <Metadata>   
+            <Name>value</Name>
+          </Metadata>
+        </Blob>
+        <BlobPrefix>
+          <Name>blob-prefix</Name>
+        </BlobPrefix>
+      </Blobs>
+      <NextMarker />
+    </EnumerationResults>
+    '''
+    if response is None or response.body is None:
+        return None
+
+    blob_list = _list()
+    list_element = ETree.fromstring(response.body)
+
+    setattr(blob_list, 'next_marker', list_element.findtext('NextMarker'))
+
+    blobs_element = list_element.find('Blobs')
+    blob_prefix_elements = blobs_element.findall('BlobPrefix')
+    if blob_prefix_elements is not None:
+        for blob_prefix_element in blob_prefix_elements:
+            blob_list.append(blob_prefix_element.findtext('Name'))
+
+    for blob_element in blobs_element.findall('Blob'):
+        blob_list.append(blob_element.findtext('Name'))
+
+    return blob_list
+
+
 def _convert_xml_to_block_list(response):
     '''
     <?xml version="1.0" encoding="utf-8"?>
@@ -450,3 +521,36 @@ def _parse_account_information(response):
     account_info.account_kind = response.headers['x-ms-account-kind']
 
     return account_info
+
+
+def _convert_xml_to_user_delegation_key(response):
+    """
+    <?xml version="1.0" encoding="utf-8"?>
+    <UserDelegationKey>
+        <SignedOid> Guid </SignedOid>
+        <SignedTid> Guid </SignedTid>
+        <SignedStart> String, formatted ISO Date </SignedStart>
+        <SignedExpiry> String, formatted ISO Date </SignedExpiry>
+        <SignedService>b</SignedService>
+        <SignedVersion> String, rest api version used to create delegation key </SignedVersion>
+        <Value>Ovg+o0K/0/2V8upg7AwlyAPCriEcOSXKuBu2Gv/PU70Y7aWDW3C2ZRmw6kYWqPWBaM1GosLkcSZkgsobAlT+Sw==</value>
+    </UserDelegationKey >
+
+    Converts xml response to UserDelegationKey class.
+    """
+
+    if response is None or response.body is None:
+        return None
+
+    delegation_key = UserDelegationKey()
+
+    key_element = ETree.fromstring(response.body)
+    delegation_key.signed_oid = key_element.findtext('SignedOid')
+    delegation_key.signed_tid = key_element.findtext('SignedTid')
+    delegation_key.signed_start = key_element.findtext('SignedStart')
+    delegation_key.signed_expiry = key_element.findtext('SignedExpiry')
+    delegation_key.signed_service = key_element.findtext('SignedService')
+    delegation_key.signed_version = key_element.findtext('SignedVersion')
+    delegation_key.value = key_element.findtext('Value')
+
+    return delegation_key

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_encryption.py

@@ -123,8 +123,8 @@ def _decrypt_blob(require_encryption, key_encryption_key, key_resolver,
     except:
         if require_encryption:
             raise ValueError(_ERROR_DATA_NOT_ENCRYPTED)
-        else:
-            return content
+
+        return content
 
     if not (encryption_data.encryption_agent.encryption_algorithm == _EncryptionAlgorithm.AES_CBC_256):
         raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM)

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/_serialization.py

@@ -4,7 +4,7 @@
 # license information.
 # --------------------------------------------------------------------------
 from xml.sax.saxutils import escape as xml_escape
-
+from datetime import date
 try:
     from xml.etree import cElementTree as ETree
 except ImportError:
@@ -13,6 +13,9 @@
     _encode_base64,
     _str,
 )
+from ..common._serialization import (
+    _to_utc_datetime,
+)
 from ..common._error import (
     _validate_not_none,
     _ERROR_START_END_NEEDED_FOR_MD5,
@@ -46,7 +49,8 @@ def _get_path(container_name=None, blob_name=None):
 
 
 def _validate_and_format_range_headers(request, start_range, end_range, start_range_required=True,
-                                       end_range_required=True, check_content_md5=False, align_to_page=False):
+                                       end_range_required=True, check_content_md5=False, align_to_page=False,
+                                       range_header_name='x-ms-range'):
     # If end range is provided, start range must be provided
     if start_range_required or end_range is not None:
         _validate_not_none('start_range', start_range)
@@ -63,9 +67,9 @@ def _validate_and_format_range_headers(request, start_range, end_range, start_ra
     # Format based on whether end_range is present
     request.headers = request.headers or {}
     if end_range is not None:
-        request.headers['x-ms-range'] = 'bytes={0}-{1}'.format(start_range, end_range)
+        request.headers[range_header_name] = 'bytes={0}-{1}'.format(start_range, end_range)
     elif start_range is not None:
-        request.headers['x-ms-range'] = "bytes={0}-".format(start_range)
+        request.headers[range_header_name] = "bytes={0}-".format(start_range)
 
     # Content MD5 can only be provided for a complete range less than 4MB in size
     if check_content_md5:
@@ -116,3 +120,34 @@ def _convert_block_list_to_xml(block_id_list):
 
     # return xml value
     return output
+
+
+def _convert_delegation_key_info_to_xml(start_time, expiry_time):
+    """
+    <?xml version="1.0" encoding="utf-8"?>
+    <KeyInfo>
+        <Start> String, formatted ISO Date </Start>
+        <Expiry> String, formatted ISO Date </Expiry>
+    </KeyInfo>
+
+    Convert key info to xml to send.
+    """
+    if start_time is None or expiry_time is None:
+        raise ValueError("delegation key start/end times are required")
+
+    key_info_element = ETree.Element('KeyInfo')
+    ETree.SubElement(key_info_element, 'Start').text = \
+        _to_utc_datetime(start_time) if isinstance(start_time, date) else start_time
+    ETree.SubElement(key_info_element, 'Expiry').text = \
+        _to_utc_datetime(expiry_time) if isinstance(expiry_time, date) else expiry_time
+
+    # Add xml declaration and serialize
+    try:
+        stream = BytesIO()
+        ETree.ElementTree(key_info_element).write(stream, xml_declaration=True, encoding='utf-8', method='xml')
+    finally:
+        output = stream.getvalue()
+        stream.close()
+
+    # return xml value
+    return output

sdk/eventhub/azure-eventhubs/azure/eventprocessorhost/vendor/storage/blob/appendblobservice.py

@@ -33,6 +33,7 @@
 )
 from ._serialization import (
     _get_path,
+    _validate_and_format_range_headers,
 )
 from ._upload_chunking import (
     _AppendBlobChunkUploader,
@@ -112,7 +113,7 @@ def __init__(self, account_name=None, account_key=None, sas_token=None, is_emula
         :param token_credential:
             A token credential used to authenticate HTTPS requests. The token value
             should be updated before its expiration.
-        :type `~..common.TokenCredential`
+        :type `~azure.storage.common.TokenCredential`
         '''
         self.blob_type = _BlobTypes.AppendBlob
         super(AppendBlobService, self).__init__(
@@ -286,6 +287,125 @@ def append_block(self, container_name, blob_name, block,
 
         return self._perform_request(request, _parse_append_block)
 
+    def append_block_from_url(self, container_name, blob_name, copy_source_url, source_range_start=None,
+                              source_range_end=None, source_content_md5=None, source_if_modified_since=None,
+                              source_if_unmodified_since=None, source_if_match=None,
+                              source_if_none_match=None, maxsize_condition=None,
+                              appendpos_condition=None, lease_id=None, if_modified_since=None,
+                              if_unmodified_since=None, if_match=None,
+                              if_none_match=None, timeout=None):
+        """
+        Creates a new block to be committed as part of a blob, where the contents are read from a source url.
+
+        :param str container_name:
+            Name of existing container.
+        :param str blob_name:
+            Name of blob.
+        :param str copy_source_url:
+            The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
+            shared access signature attached.
+        :param int source_range_start:
+            This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
+        :param int source_range_end:
+            This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source.
+        :param str source_content_md5:
+            If given, the service will calculate the MD5 hash of the block content and compare against this value.
+        :param datetime source_if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the source resource has been modified since the specified time.
+        :param datetime source_if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the source resource has not been modified since the specified date/time.
+        :param str source_if_match:
+            An ETag value, or the wildcard character (*). Specify this header to perform
+            the operation only if the source resource's ETag matches the value specified.
+        :param str source_if_none_match:
+            An ETag value, or the wildcard character (*). Specify this header
+            to perform the operation only if the source resource's ETag does not match
+            the value specified. Specify the wildcard character (*) to perform
+            the operation only if the source resource does not exist, and fail the
+            operation if it does exist.
+        :param int maxsize_condition:
+            Optional conditional header. The max length in bytes permitted for
+            the append blob. If the Append Block operation would cause the blob
+            to exceed that limit or if the blob size is already greater than the
+            value specified in this header, the request will fail with
+            MaxBlobSizeConditionNotMet error (HTTP status code 412 - Precondition Failed).
+        :param int appendpos_condition:
+            Optional conditional header, used only for the Append Block operation.
+            A number indicating the byte offset to compare. Append Block will
+            succeed only if the append position is equal to this number. If it
+            is not, the request will fail with the
+            AppendPositionConditionNotMet error
+            (HTTP status code 412 - Precondition Failed).
+        :param str lease_id:
+            Required if the blob has an active lease.
+        :param datetime if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the resource has been modified since the specified time.
+        :param datetime if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the resource has not been modified since the specified date/time.
+        :param str if_match:
+            An ETag value, or the wildcard character (*). Specify this header to perform
+            the operation only if the resource's ETag matches the value specified.
+        :param str if_none_match:
+            An ETag value, or the wildcard character (*). Specify this header
+            to perform the operation only if the resource's ETag does not match
+            the value specified. Specify the wildcard character (*) to perform
+            the operation only if the resource does not exist, and fail the
+            operation if it does exist.
+        :param int timeout:
+            The timeout parameter is expressed in seconds.
+        """
+        _validate_encryption_unsupported(self.require_encryption, self.key_encryption_key)
+        _validate_not_none('container_name', container_name)
+        _validate_not_none('blob_name', blob_name)
+        _validate_not_none('copy_source_url', copy_source_url)
+
+        request = HTTPRequest()
+        request.method = 'PUT'
+        request.host_locations = self._get_host_locations()
+        request.path = _get_path(container_name, blob_name)
+        request.query = {
+            'comp': 'appendblock',
+            'timeout': _int_to_str(timeout),
+        }
+        request.headers = {
+            'x-ms-copy-source': copy_source_url,
+            'x-ms-source-content-md5': source_content_md5,
+            'x-ms-source-if-Modified-Since': _datetime_to_utc_string(source_if_modified_since),
+            'x-ms-source-if-Unmodified-Since': _datetime_to_utc_string(source_if_unmodified_since),
+            'x-ms-source-if-Match': _to_str(source_if_match),
+            'x-ms-source-if-None-Match': _to_str(source_if_none_match),
+            'x-ms-blob-condition-maxsize': _to_str(maxsize_condition),
+            'x-ms-blob-condition-appendpos': _to_str(appendpos_condition),
+            'x-ms-lease-id': _to_str(lease_id),
+            'If-Modified-Since': _datetime_to_utc_string(if_modified_since),
+            'If-Unmodified-Since': _datetime_to_utc_string(if_unmodified_since),
+            'If-Match': _to_str(if_match),
+            'If-None-Match': _to_str(if_none_match)
+        }
+
+        _validate_and_format_range_headers(request, source_range_start, source_range_end,
+                                           start_range_required=False,
+                                           end_range_required=False,
+                                           range_header_name="x-ms-source-range")
+
+        return self._perform_request(request, _parse_append_block)
+
     # ----Convenience APIs----------------------------------------------
 
     def append_blob_from_path(