From 6f87ce80aced0b504434b7cb8b27e332f629bdfe Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:40:12 -0400 Subject: [PATCH 1/9] Try another retry --- codecov/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index f7f5126e..10599c84 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -287,6 +287,10 @@ def generate_toc(root): return str(res).strip() or "" +def retry_upload(url, request_method, **kwargs): + request_method(url, **kwargs) + + def main(*argv, **kwargs): root = os.getcwd() @@ -1100,8 +1104,9 @@ def main(*argv, **kwargs): if "s3" not in codecov.disable: try: write(" Pinging Codecov...") - res = requests.post( + res = retry_upload( "%s/upload/v4?%s" % (codecov.url, urlargs), + requests.post, verify=codecov.cacert, headers={ "Accept": "text/plain", From dbb29fbd3f3ea110344d0775fdc8531683380f12 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:41:57 -0400 Subject: [PATCH 2/9] return --- codecov/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index 10599c84..f5f479b6 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -288,7 +288,7 @@ def generate_toc(root): def retry_upload(url, request_method, **kwargs): - request_method(url, **kwargs) + return request_method(url, **kwargs) def main(*argv, **kwargs): From 3a11bdbbc9aba80b55943e4dc3c52556d2ed041f Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:45:17 -0400 Subject: [PATCH 3/9] Retry all the things --- codecov/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index f5f479b6..f8a96dff 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -1123,8 +1123,9 @@ def main(*argv, **kwargs): result, upload_url = res[0], res[1] write(" Uploading to S3...") - s3 = requests.put( + s3 = retry_upload( upload_url, + requests.put, verify=codecov.cacert, data=reports_gzip, headers={ @@ -1145,8 +1146,9 @@ def main(*argv, **kwargs): write(" Uploading to Codecov...") # just incase, try traditional upload - res = requests.post( + res = retry_upload( "%s/upload/v2?%s" % (codecov.url, urlargs), + requests.post, verify=codecov.cacert, data=reports_gzip, headers={ From 241b4fc8a1d80b65824d4fb8f8082ba077d5f502 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:51:10 -0400 Subject: [PATCH 4/9] Bail from the original retries --- codecov/__init__.py | 79 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index f8a96dff..0baee55e 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -1098,51 +1098,48 @@ def main(*argv, **kwargs): write(" Compressed contents to {0} bytes".format(len(reports_gzip))) s3 = None - trys = 0 - while trys < 3: - trys += 1 - if "s3" not in codecov.disable: - try: - write(" Pinging Codecov...") - res = retry_upload( - "%s/upload/v4?%s" % (codecov.url, urlargs), - requests.post, + if "s3" not in codecov.disable: + try: + write(" Pinging Codecov...") + res = retry_upload( + "%s/upload/v4?%s" % (codecov.url, urlargs), + requests.post, + verify=codecov.cacert, + headers={ + "Accept": "text/plain", + "X-Reduced-Redundancy": "false", + "X-Content-Type": "application/x-gzip", + }, + ) + if res.status_code in (400, 406): + raise Exception(res.text) + + elif res.status_code < 500: + assert res.status_code == 200 + res = res.text.strip().split() + result, upload_url = res[0], res[1] + + write(" Uploading to S3...") + s3 = retry_upload( + upload_url, + requests.put, verify=codecov.cacert, + data=reports_gzip, headers={ - "Accept": "text/plain", - "X-Reduced-Redundancy": "false", - "X-Content-Type": "application/x-gzip", + "Content-Type": "application/x-gzip", + "Content-Encoding": "gzip", }, ) - if res.status_code in (400, 406): - raise Exception(res.text) - - elif res.status_code < 500: - assert res.status_code == 200 - res = res.text.strip().split() - result, upload_url = res[0], res[1] - - write(" Uploading to S3...") - s3 = retry_upload( - upload_url, - requests.put, - verify=codecov.cacert, - data=reports_gzip, - headers={ - "Content-Type": "application/x-gzip", - "Content-Encoding": "gzip", - }, - ) - s3.raise_for_status() - assert s3.status_code == 200 - write(" " + result) - break - else: - # try again - continue - - except AssertionError: - write(" Direct to s3 failed. Using backup v2 endpoint.") + s3.raise_for_status() + assert s3.status_code == 200 + write(" " + result) + break + else: + # try again + continue + + except AssertionError: + write(" Direct to s3 failed. Using backup v2 endpoint.") write(" Uploading to Codecov...") # just incase, try traditional upload From 502e26c2c7026729d9dba5d0f2e4a022b58079d2 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:54:20 -0400 Subject: [PATCH 5/9] do real retries --- codecov/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index 0baee55e..698776fd 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -287,8 +287,12 @@ def generate_toc(root): return str(res).strip() or "" -def retry_upload(url, request_method, **kwargs): - return request_method(url, **kwargs) +def retry_upload(url, request_method, retries=3, break_codes=(200, ), **kwargs): + for _ in range(retries): + res = request_method(url, **kwargs) + if res.status_code in break_codes: + return res + return res def main(*argv, **kwargs): @@ -1104,6 +1108,7 @@ def main(*argv, **kwargs): res = retry_upload( "%s/upload/v4?%s" % (codecov.url, urlargs), requests.post, + break_codes=(200, 400, 406) verify=codecov.cacert, headers={ "Accept": "text/plain", From 706429b8808ccc8a7a5beeac1f89d0d7329fe8c7 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:56:36 -0400 Subject: [PATCH 6/9] fixes --- codecov/__init__.py | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index 698776fd..b8f575dd 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -1102,6 +1102,7 @@ def main(*argv, **kwargs): write(" Compressed contents to {0} bytes".format(len(reports_gzip))) s3 = None + success = False if "s3" not in codecov.disable: try: write(" Pinging Codecov...") @@ -1138,35 +1139,30 @@ def main(*argv, **kwargs): s3.raise_for_status() assert s3.status_code == 200 write(" " + result) - break - else: - # try again - continue + success = True except AssertionError: write(" Direct to s3 failed. Using backup v2 endpoint.") - write(" Uploading to Codecov...") # just incase, try traditional upload - res = retry_upload( - "%s/upload/v2?%s" % (codecov.url, urlargs), - requests.post, - verify=codecov.cacert, - data=reports_gzip, - headers={ - "Accept": "text/plain", - "Content-Type": "application/x-gzip", - "Content-Encoding": "gzip", - }, - ) - if res.status_code < 500: - write(" " + res.text) - res.raise_for_status() - result = res.text - return - - write(" Retrying... in %ds" % (trys * 30)) - sleep(trys * 30) + if not success: + write(" Uploading to Codecov...") + res = retry_upload( + "%s/upload/v2?%s" % (codecov.url, urlargs), + requests.post, + verify=codecov.cacert, + data=reports_gzip, + headers={ + "Accept": "text/plain", + "Content-Type": "application/x-gzip", + "Content-Encoding": "gzip", + }, + ) + if res.status_code < 500: + write(" " + res.text) + res.raise_for_status() + result = res.text + return except Exception as e: write("Error: " + str(e)) From 545e6c3a2fa8794f86b762cc04e43cdd546d4cc2 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 17:59:40 -0400 Subject: [PATCH 7/9] syntax --- codecov/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index b8f575dd..e2677729 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -1109,7 +1109,7 @@ def main(*argv, **kwargs): res = retry_upload( "%s/upload/v4?%s" % (codecov.url, urlargs), requests.post, - break_codes=(200, 400, 406) + break_codes=(200, 400, 406), verify=codecov.cacert, headers={ "Accept": "text/plain", From afd2f6391d908ced1d7be0779edc42341d01b0f4 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 18:02:03 -0400 Subject: [PATCH 8/9] black --- codecov/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index e2677729..51039bdd 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -287,7 +287,7 @@ def generate_toc(root): return str(res).strip() or "" -def retry_upload(url, request_method, retries=3, break_codes=(200, ), **kwargs): +def retry_upload(url, request_method, retries=3, break_codes=(200,), **kwargs): for _ in range(retries): res = request_method(url, **kwargs) if res.status_code in break_codes: From 0b340a0d48c4012e2753e315f780dda19741a147 Mon Sep 17 00:00:00 2001 From: Thomas Hu Date: Thu, 11 Jun 2020 18:04:20 -0400 Subject: [PATCH 9/9] Remove unnecessary var declaration --- codecov/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/codecov/__init__.py b/codecov/__init__.py index 51039bdd..046f4cef 100644 --- a/codecov/__init__.py +++ b/codecov/__init__.py @@ -1101,7 +1101,6 @@ def main(*argv, **kwargs): reports_gzip = gzip_worker.compress(reports) + gzip_worker.flush() write(" Compressed contents to {0} bytes".format(len(reports_gzip))) - s3 = None success = False if "s3" not in codecov.disable: try: