diff --git a/Dockerfile b/Dockerfile index e7042abf4..9f87bec2b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ USER nobody CMD gunicorn \ --workers 2 \ --threads 1 \ + --timeout 100 \ --access-logfile '-' \ --bind 0.0.0.0:8000 \ backend.wsgi diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 3f53a470b..e53624e2f 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -215,3 +215,6 @@ def check_ip_range(ipr): # `django-registration-redux` 3rd party app settings. INCLUDE_REGISTER_URL = False + +# Retry to connect to DB (after receiving a connection error) within 60 seconds. +DB_RETRY_TO_CONNECT_SEC = 60 diff --git a/backend/backend/utils.py b/backend/backend/utils.py new file mode 100644 index 000000000..69ce4c62b --- /dev/null +++ b/backend/backend/utils.py @@ -0,0 +1,61 @@ +from time import sleep, time + +from django.conf import settings + +from psycopg2 import OperationalError + + +def ensure_connection_with_retries(self): + """ + A function supposed to be used for patching the standard django database connection class' method + in order to try to connect to DB multiple times using exponential backoff algorithm. + After each consecutive connection error it waits 1, 2, 4, 8, 16, 32, ... seconds until success or + the end of time allowed to spend on reconnection attempts. + """ + + def custom_properties_cleanup(obj): + delattr(obj, '_is_connecting') + if hasattr(obj, "_connection_retry"): + delattr(obj, '_connection_retry') + + if self.connection is not None and hasattr(self.connection, 'closed') and self.connection.closed: + self.connection = None + + if self.connection is None and not hasattr(self, '_is_connecting'): + with self.wrap_database_errors: + self._is_connecting = True + try: + self.connect() + except OperationalError as e: + # We need to reconnect only after particular OperationalError types. + if e.args and (e.args[0].startswith('could not connect to server: Connection refused') or + e.args[0].startswith('could not translate host name')): + # Connection error. + if not hasattr(self, "_connection_retry"): + self._connection_retry = 0 + self._stop_trying_at = time() + settings.DB_RETRY_TO_CONNECT_SEC + if time() > self._stop_trying_at: # Stop trying to reconnect. + custom_properties_cleanup(self) + raise + else: + seconds = 0 + # We're gonna make the last try at the very end of allowed time. + while time() < self._stop_trying_at: + sleep(1) + seconds += 1 + if seconds >= 2 ** self._connection_retry: + break + self._connection_retry += 1 + self.connection = None + delattr(self, '_is_connecting') + self.ensure_connection() + else: + # Other types of OperationalError. + custom_properties_cleanup(self) + raise + except Exception: + # Other errors. + custom_properties_cleanup(self) + raise + else: + custom_properties_cleanup(self) diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py index 0f3212d3f..442444528 100644 --- a/backend/backend/wsgi.py +++ b/backend/backend/wsgi.py @@ -10,7 +10,16 @@ import os from django.core.wsgi import get_wsgi_application +from django.db.backends.base import base as django_db_base + +from .utils import ensure_connection_with_retries os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') +# Patch the standard django database connection class' method in order to try to +# connect to DB multiple times using exponential backoff algorithm until success +# or the end of time allowed to spend on reconnection attempts +# (settings.DB_RETRY_TO_CONNECT_SEC). +django_db_base.BaseDatabaseWrapper.ensure_connection = ensure_connection_with_retries + application = get_wsgi_application()