diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index c02abcb0f..cebc4fe7c 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -503,8 +503,10 @@ def __init__(self, run_args): self.requests_session = requests.Session() adapter = requests.adapters.HTTPAdapter( max_retries=Retry( - total=3, - backoff_factor=1, + total=5, + backoff_factor=2, + status_forcelist=[502, 503, 504], + allowed_methods=["PATCH", "GET", "PUT"], ) ) self.requests_session.mount("http://", adapter) @@ -618,14 +620,15 @@ def _update_submission(self, data): url = f"{self.submissions_api_url}/submissions/{self.submission_id}/" data["secret"] = self.secret - logger.info(f"Updating submission @ {url} with data = {data}") + logger.info(f"Updating submission @ {url}") resp = self.requests_session.patch(url, data=data, timeout=150) if resp.status_code == 200: logger.info("Submission updated successfully!") else: logger.error( - f"Submission patch failed with status = {resp.status_code}, and response = \n{resp.content}" + f"Submission patch failed with status = {resp.status_code}, " + f"and response = \n{resp.content}" ) raise SubmissionException("Failure updating submission data.") @@ -639,8 +642,9 @@ def _update_status(self, status, extra_information=None): try: self._update_submission(data) except Exception as e: - # Always catch exception and never raise error logger.exception(f"Failed to update submission status to {status}: {e}") + if status in ("Finished", "Failed"): + raise def _get_container_image(self, image_name): logger.info("Running pull for image: {}".format(image_name))