From 717057edef5b3b7f775e32458e8c8fd0183f0307 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 24 Sep 2021 11:49:52 -0700 Subject: [PATCH 1/4] storage: add configure_retries sample --- .../cloud-client/storage_configure_retries.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 storage/cloud-client/storage_configure_retries.py diff --git a/storage/cloud-client/storage_configure_retries.py b/storage/cloud-client/storage_configure_retries.py new file mode 100644 index 00000000000..de8435b0ce1 --- /dev/null +++ b/storage/cloud-client/storage_configure_retries.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# [START storage_configure_retries] +from google.cloud import storage +from google.cloud.storage.retry import DEFAULT_RETRY + + +def configure_retries(bucket_name, blob_name): + """Configures retries with customizations.""" + # The ID of your GCS bucket + # bucket_name = "your-bucket-name" + # The ID of your GCS object + # blob_name = "your-object-name" + + storage_client = storage.Client() + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + # Customize Retry with a deadline of 500 seconds instead of the default 120 seconds. + # Customize Retry with a wait time multiplier per iteration of 3.0 instead of the default 2.0. + modified_retry = DEFAULT_RETRY.with_deadline(500.0) + modified_retry = modified_retry.with_delay(multiplier=3.0) + + # blob.delete() uses DEFAULT_RETRY_IF_GENERATION_SPECIFIED by default. + # Override with modified_retry so that the function retries even if the generation number is not specified. + print( + "The following library method is customized to be retried according to the following configurations:" + ) + print(modified_retry.__str__()) + + blob.delete(retry=modified_retry) + print("Blob {} deleted with a customized retry strategy.".format(blob_name)) + + +# [END storage_configure_retries] + + +if __name__ == "__main__": + configure_retries(bucket_name=sys.argv[1], blob_name=sys.argv[2]) From 2373c1631ac718ee6d8e77a670e1e2554e84450e Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Fri, 24 Sep 2021 11:57:19 -0700 Subject: [PATCH 2/4] add test for storage_configure_retries sample --- storage/cloud-client/snippets_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/storage/cloud-client/snippets_test.py b/storage/cloud-client/snippets_test.py index f4d7987fe94..2b6d0a817a9 100644 --- a/storage/cloud-client/snippets_test.py +++ b/storage/cloud-client/snippets_test.py @@ -27,6 +27,7 @@ import storage_change_default_storage_class import storage_change_file_storage_class import storage_compose_file +import storage_configure_retries import storage_copy_file import storage_copy_file_archived_generation import storage_cors_configuration @@ -498,3 +499,13 @@ def test_list_blobs_archived_generation(test_blob, capsys): ) out, _ = capsys.readouterr() assert str(test_blob.generation) in out + + +def test_storage_configure_retries(test_blob, capsys): + storage_configure_retries.configure_retries(test_blob.bucket.name, test_blob.name) + + # This simply checks if the retry configurations were set and printed as intended. + out, _ = capsys.readouterr() + assert "The following library method is customized to be retried" in out + assert "_should_retry" in out + assert "initial=1.0, maximum=60.0, multiplier=3.0, deadline=500.0" in out From 00688c1c86228f39c1b1c06d9cd03374372d7274 Mon Sep 17 00:00:00 2001 From: Cathy Ouyang Date: Mon, 11 Oct 2021 11:32:49 -0700 Subject: [PATCH 3/4] revise comments per discussion with TW --- storage/cloud-client/storage_configure_retries.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/storage/cloud-client/storage_configure_retries.py b/storage/cloud-client/storage_configure_retries.py index de8435b0ce1..3911cb50dc4 100644 --- a/storage/cloud-client/storage_configure_retries.py +++ b/storage/cloud-client/storage_configure_retries.py @@ -16,6 +16,12 @@ import sys +"""Sample that configures retries on an operation call. +This sample is used on this page: + https://cloud.google.com/storage/docs/retry-strategy +For more information, see README.md. +""" + # [START storage_configure_retries] from google.cloud import storage from google.cloud.storage.retry import DEFAULT_RETRY @@ -32,17 +38,16 @@ def configure_retries(bucket_name, blob_name): bucket = storage_client.bucket(bucket_name) blob = bucket.blob(blob_name) - # Customize Retry with a deadline of 500 seconds instead of the default 120 seconds. - # Customize Retry with a wait time multiplier per iteration of 3.0 instead of the default 2.0. + # Customize retry with a deadline of 500 seconds (default=120 seconds). modified_retry = DEFAULT_RETRY.with_deadline(500.0) + # Customize retry with a wait time multiplier per iteration of 3.0 (default=2.0). modified_retry = modified_retry.with_delay(multiplier=3.0) # blob.delete() uses DEFAULT_RETRY_IF_GENERATION_SPECIFIED by default. # Override with modified_retry so that the function retries even if the generation number is not specified. print( - "The following library method is customized to be retried according to the following configurations:" + f"The following library method is customized to be retried according to the following configurations: {modified_retry}" ) - print(modified_retry.__str__()) blob.delete(retry=modified_retry) print("Blob {} deleted with a customized retry strategy.".format(blob_name)) From 4574913172a47e40e126fe808dac9cccc637cc7f Mon Sep 17 00:00:00 2001 From: Bonnie Chan <52431539+cbonnie@users.noreply.github.com> Date: Wed, 13 Oct 2021 14:14:17 -0400 Subject: [PATCH 4/4] Shortening code comment and line wrapping --- storage/cloud-client/storage_configure_retries.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/storage/cloud-client/storage_configure_retries.py b/storage/cloud-client/storage_configure_retries.py index 3911cb50dc4..68b1e1fff7d 100644 --- a/storage/cloud-client/storage_configure_retries.py +++ b/storage/cloud-client/storage_configure_retries.py @@ -44,7 +44,8 @@ def configure_retries(bucket_name, blob_name): modified_retry = modified_retry.with_delay(multiplier=3.0) # blob.delete() uses DEFAULT_RETRY_IF_GENERATION_SPECIFIED by default. - # Override with modified_retry so that the function retries even if the generation number is not specified. + # Override with modified_retry so the function retries even if the generation + # number is not specified. print( f"The following library method is customized to be retried according to the following configurations: {modified_retry}" )