testing: harden systest teardowns vs transient GCS errors

noxfile.py

@@ -24,8 +24,12 @@
 )
 GOOGLE_AUTH = 'google-auth >= 0.10.0'
 
+DEFAULT_PYTHON_VERSION = "3.8"
+SYSTEM_TEST_PYTHON_VERSIONS = ["2.7", "3.8"]
+UNIT_TEST_PYTHON_VERSIONS = ["2.7", "3.5", "3.6", "3.7", "3.8"]
 
-@nox.session(python=['2.7', '3.5', '3.6', '3.7', '3.8'])
+
+@nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
 def unit(session):
     """Run the unit test suite."""
 
@@ -50,7 +54,7 @@ def unit(session):
     )
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def docs(session):
     """Build the docs for this library."""
 
@@ -71,7 +75,7 @@ def docs(session):
         os.path.join("docs", "_build", "html", ""),
     )
 
-@nox.session(python="3.8")
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def docfx(session):
     """Build the docfx yaml files for this library."""
 
@@ -104,7 +108,7 @@ def docfx(session):
     )
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def doctest(session):
     """Run the doctests."""
     session.install("-e", ".[requests]")
@@ -129,7 +133,7 @@ def doctest(session):
     )
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def lint(session):
     """Run flake8.
 
@@ -146,21 +150,21 @@ def lint(session):
     session.run("black", "--check", os.path.join("google", "resumable_media"), "tests")
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def lint_setup_py(session):
     """Verify that setup.py is valid (including RST check)."""
     session.install('docutils', 'Pygments')
     session.run(
         'python', 'setup.py', 'check', '--restructuredtext', '--strict')
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def blacken(session):
     session.install("black")
     session.run("black", os.path.join("google", "resumable_media"), "tests")
 
 
-@nox.session(python=['2.7', '3.8'])
+@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS)
 def system(session):
     """Run the system test suite."""
 
@@ -178,7 +182,7 @@ def system(session):
 
     # Install all test dependencies, then install this package into the
     # virutalenv's dist-packages.
-    session.install('mock', 'pytest', GOOGLE_AUTH)
+    session.install('mock', 'pytest', GOOGLE_AUTH, 'google-cloud-testutils')
     session.install('-e', '.[requests]')
 
     # Run py.test against the system tests.
@@ -189,7 +193,7 @@ def system(session):
     )
 
 
-@nox.session(python='3.8')
+@nox.session(python=DEFAULT_PYTHON_VERSION)
 def cover(session):
     """Run the final coverage report.
 

tests/system/requests/conftest.py

@@ -37,7 +37,7 @@ def ensure_bucket(transport):
 
 
 def cleanup_bucket(transport):
-    del_response = transport.delete(utils.BUCKET_URL)
+    del_response = utils.retry_transient_errors(transport.delete)(utils.BUCKET_URL)
 
     if not del_response.ok:
         raise ValueError("{}: {}".format(del_response.status_code, del_response.reason))

tests/system/requests/test_upload.py

@@ -56,7 +56,7 @@ def add_cleanup(blob_name, transport):
 
     for blob_name, transport in to_delete:
         metadata_url = utils.METADATA_URL_TEMPLATE.format(blob_name=blob_name)
-        response = transport.delete(metadata_url)
+        response = utils.retry_transient_errors(transport.delete)(metadata_url)
         assert response.status_code == http_client.NO_CONTENT
 
 

tests/system/utils.py

@@ -16,6 +16,8 @@
 import hashlib
 import time
 
+from test_utils.retry import RetryResult
+
 
 BUCKET_NAME = u"grpm-systest-{}".format(int(1000 * time.time()))
 BUCKET_POST_URL = u"https://www.googleapis.com/storage/v1/b/"
@@ -44,6 +46,20 @@
 )
 
 
+_RETRYABLE_CODES = [
+    409,  # Conflict
+    429,  # TooManyRequests
+    503,  # ServiceUnavailable
+]
+
+
+def _not_retryable(response):
+    return response.status_code not in _RETRYABLE_CODES
+
+
+retry_transient_errors = RetryResult(_not_retryable)
+
+
 def get_encryption_headers(key=ENCRYPTION_KEY):
     """Builds customer-supplied encryption key headers