-
Notifications
You must be signed in to change notification settings - Fork 3.3k
[Blob] Added upload blob from url feature #15027
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
5e813e2
a3c4168
4adabd4
042387e
0e6dfd8
cd390f1
7e20151
0343733
2849f01
66a572c
53fecd5
9fac13f
7c933ef
821771a
f5a0e52
f28551d
9313c7c
64d9349
7547365
83df6d0
e9ceb3e
e879ede
3a3a6bd
8de4ceb
fdd2f5f
90532f3
e177b34
8708de0
8663c23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -52,7 +52,7 @@ | |
| from ._upload_helpers import ( | ||
| upload_block_blob, | ||
| upload_append_blob, | ||
| upload_page_blob) | ||
| upload_page_blob, _any_conditions) | ||
| from ._models import BlobType, BlobBlock, BlobProperties, BlobQueryError | ||
| from ._download import StorageStreamDownloader | ||
| from ._lease import BlobLeaseClient | ||
|
|
@@ -406,6 +406,146 @@ def _upload_blob_options( # pylint:disable=too-many-statements | |
| raise ValueError("Unsupported BlobType: {}".format(blob_type)) | ||
| return kwargs | ||
|
|
||
| def _upload_blob_from_url_options(self, source_url, **kwargs): | ||
| # type: (...) -> Dict[str, Any] | ||
| tier = kwargs.pop('standard_blob_tier', None) | ||
| overwrite = kwargs.pop('overwrite', False) | ||
| content_settings = kwargs.pop('content_settings', None) | ||
| if content_settings: | ||
| kwargs['blob_http_headers'] = BlobHTTPHeaders( | ||
| blob_cache_control=content_settings.cache_control, | ||
| blob_content_type=content_settings.content_type, | ||
| blob_content_md5=None, | ||
| blob_content_encoding=content_settings.content_encoding, | ||
| blob_content_language=content_settings.content_language, | ||
| blob_content_disposition=content_settings.content_disposition | ||
| ) | ||
| cpk = kwargs.pop('cpk', None) | ||
| cpk_info = None | ||
| if cpk: | ||
| if self.scheme.lower() != 'https': | ||
| raise ValueError("Customer provided encryption key must be used over HTTPS.") | ||
| cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash, | ||
| encryption_algorithm=cpk.algorithm) | ||
|
|
||
| options = { | ||
| 'content_length': 0, | ||
| 'copy_source_blob_properties': kwargs.pop('include_source_blob_properties', True), | ||
| 'source_content_md5': kwargs.pop('source_content_md5', None), | ||
| 'copy_source': source_url, | ||
|
tasherif-msft marked this conversation as resolved.
|
||
| 'modified_access_conditions': get_modify_conditions(kwargs), | ||
| 'blob_tags_string': serialize_blob_tags_header(kwargs.pop('tags', None)), | ||
| 'cls': return_response_headers, | ||
| 'lease_access_conditions': get_access_conditions(kwargs.pop('destination_lease', None)), | ||
| 'tier': tier.value if tier else None, | ||
| 'source_modified_access_conditions': get_source_conditions(kwargs), | ||
| 'cpk_info': cpk_info, | ||
| 'cpk_scope_info': get_cpk_scope_info(kwargs) | ||
| } | ||
| options.update(kwargs) | ||
| if not overwrite and not _any_conditions(**options): # pylint: disable=protected-access | ||
| options['modified_access_conditions'].if_none_match = '*' | ||
| return options | ||
|
|
||
| @distributed_trace | ||
| def upload_blob_from_url(self, source_url, **kwargs): | ||
|
tasherif-msft marked this conversation as resolved.
|
||
| # type: (str, Any) -> Dict[str, Any] | ||
| """ | ||
| Creates a new Block Blob where the content of the blob is read from a given URL. | ||
| The content of an existing blob is overwritten with the new blob. | ||
|
|
||
| :param str source_url: | ||
| A URL of up to 2 KB in length that specifies a file or blob. | ||
| The value should be URL-encoded as it would appear in a request URI. | ||
| If the source is in another account, the source must either be public | ||
| or must be authenticated via a shared access signature. If the source | ||
| is public, no authentication is required. | ||
| Examples: | ||
| https://myaccount.blob.core.windows.net/mycontainer/myblob | ||
|
|
||
| https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot=<DateTime> | ||
|
|
||
| https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken | ||
| :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data. | ||
| If True, upload_blob will overwrite the existing data. If set to False, the | ||
| operation will fail with ResourceExistsError. | ||
| :keyword bool include_source_blob_properties: | ||
| Indicates if properties from the source blob should be copied. Defaults to True. | ||
| :keyword tags: | ||
| Name-value pairs associated with the blob as tag. Tags are case-sensitive. | ||
| The tag set may contain at most 10 tags. Tag keys must be between 1 and 128 characters, | ||
| and tag values must be between 0 and 256 characters. | ||
| Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9), | ||
| space (` `), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_) | ||
| :paramtype tags: dict(str, str) | ||
| :keyword bytearray source_content_md5: | ||
| Specify the md5 that is used to verify the integrity of the source bytes. | ||
| :keyword ~datetime.datetime source_if_modified_since: | ||
| A DateTime value. Azure expects the date value passed in to be UTC. | ||
| If timezone is included, any non-UTC datetimes will be converted to UTC. | ||
| If a date is passed in without timezone info, it is assumed to be UTC. | ||
| Specify this header to perform the operation only | ||
| if the source resource has been modified since the specified time. | ||
| :keyword ~datetime.datetime source_if_unmodified_since: | ||
| A DateTime value. Azure expects the date value passed in to be UTC. | ||
| If timezone is included, any non-UTC datetimes will be converted to UTC. | ||
| If a date is passed in without timezone info, it is assumed to be UTC. | ||
| Specify this header to perform the operation only if | ||
| the source resource has not been modified since the specified date/time. | ||
| :keyword str source_etag: | ||
| The source ETag value, or the wildcard character (*). Used to check if the resource has changed, | ||
| and act according to the condition specified by the `match_condition` parameter. | ||
| :keyword ~azure.core.MatchConditions source_match_condition: | ||
| The source match condition to use upon the etag. | ||
| :keyword ~datetime.datetime if_modified_since: | ||
| A DateTime value. Azure expects the date value passed in to be UTC. | ||
| If timezone is included, any non-UTC datetimes will be converted to UTC. | ||
| If a date is passed in without timezone info, it is assumed to be UTC. | ||
| Specify this header to perform the operation only | ||
| if the resource has been modified since the specified time. | ||
| :keyword ~datetime.datetime if_unmodified_since: | ||
| A DateTime value. Azure expects the date value passed in to be UTC. | ||
| If timezone is included, any non-UTC datetimes will be converted to UTC. | ||
| If a date is passed in without timezone info, it is assumed to be UTC. | ||
| Specify this header to perform the operation only if | ||
| the resource has not been modified since the specified date/time. | ||
| :keyword str etag: | ||
| The destination ETag value, or the wildcard character (*). Used to check if the resource has changed, | ||
| and act according to the condition specified by the `match_condition` parameter. | ||
| :keyword ~azure.core.MatchConditions match_condition: | ||
| The destination match condition to use upon the etag. | ||
|
Comment on lines
+500
to
+516
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we prefix them with "destination_" ?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It appears we're using |
||
| :keyword destination_lease: | ||
| The lease ID specified for this header must match the lease ID of the | ||
| destination blob. If the request does not include the lease ID or it is not | ||
| valid, the operation fails with status code 412 (Precondition Failed). | ||
| :paramtype destination_lease: ~azure.storage.blob.BlobLeaseClient or str | ||
| :keyword int timeout: | ||
| The timeout parameter is expressed in seconds. | ||
| :keyword ~azure.storage.blob.ContentSettings content_settings: | ||
| ContentSettings object used to set blob properties. Used to set content type, encoding, | ||
| language, disposition, md5, and cache control. | ||
| :keyword ~azure.storage.blob.CustomerProvidedEncryptionKey cpk: | ||
| Encrypts the data on the service-side with the given key. | ||
| Use of customer-provided keys must be done over HTTPS. | ||
| As the encryption key itself is provided in the request, | ||
| a secure connection must be established to transfer the key. | ||
| :keyword str encryption_scope: | ||
| A predefined encryption scope used to encrypt the data on the service. An encryption | ||
| scope can be created using the Management API and referenced here by name. If a default | ||
| encryption scope has been defined at the container, this value will override it if the | ||
| container-level scope is configured to allow overrides. Otherwise an error will be raised. | ||
| :keyword ~azure.storage.blob.StandardBlobTier standard_blob_tier: | ||
| A standard blob tier value to set the blob to. For this version of the library, | ||
| this is only applicable to block blobs on standard storage accounts. | ||
| """ | ||
| options = self._upload_blob_from_url_options( | ||
| source_url=self._encode_source_url(source_url), | ||
| **kwargs) | ||
| try: | ||
| return self._client.block_blob.put_blob_from_url(**options) | ||
| except StorageErrorException as error: | ||
| process_storage_error(error) | ||
|
|
||
| @distributed_trace | ||
| def upload_blob( # pylint: disable=too-many-locals | ||
| self, data, # type: Union[Iterable[AnyStr], IO[AnyStr]] | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.