From 59c9660ed7b104987e185a40dbbd5b24e48f46cb Mon Sep 17 00:00:00 2001 From: didayolo Date: Fri, 21 Apr 2023 17:06:46 +0200 Subject: [PATCH 1/2] Add retries --- src/tests/functional/test_submissions.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index a8451797c..fd180e1f1 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -13,12 +13,27 @@ SHORT_WAIT = 0.2 +def retry_test(assertion, max_retries=3): + retries = 0 + success = False + while retries < max_retries and not success: + try: + assert assertion + except AssertionError: + retries += 1 + if retries < max_retries: + print(f"Retrying the test, attempt {retries} of {max_retries}...") + else: + print(f"Test failed after {max_retries} attempts.") + raise + + class TestSubmissions(SeleniumTestCase): def setUp(self): super().setUp() self.user = UserFactory(password='test') - def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submission_zip_path, expected_submission_output, has_solutions=True, timeout=999): + def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submission_zip_path, expected_submission_output, has_solutions=True, timeout=600): """Creates a competition and runs a submission inside it, waiting for expected output to appear in submission realtime output panel. @@ -50,9 +65,10 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio assert self.find_text_in_class('.submission-output-container .title', f"Running {submission_zip_path}", timeout=timeout) # Inside the accordion the output is being streamed - self.find('.submission-output-container .title').click() self.wait(SHORT_WAIT) - assert self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout) + self.find('.submission-output-container .title').click() + # 3 retries, this tests randomly fails, maybe because a problem of synchronization + retry_test(self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout), max_retries=3) # The submission table lists our submission! assert self.find('submission-manager#user-submission-table table tbody tr:nth-of-type(1) td:nth-of-type(2)').text == submission_zip_path @@ -75,7 +91,7 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio assert Decimal(self.find('leaderboards table tbody tr:nth-of-type(1) td:nth-of-type(3)').text) == prediction_score def test_v15_submission_end_to_end(self): - self._run_submission_and_add_to_leaderboard('competition_15.zip', 'submission_15.zip', '*** prediction_score', has_solutions=False, timeout=2000) + self._run_submission_and_add_to_leaderboard('competition_15.zip', 'submission_15.zip', '*** prediction_score', has_solutions=False) def test_v18_submission_end_to_end(self): self._run_submission_and_add_to_leaderboard('competition_18.zip', 'submission_18.zip', 'results', has_solutions=False) From 84aabfab5cb24fc12b10e7acb7353196da5cc789 Mon Sep 17 00:00:00 2001 From: didayolo Date: Fri, 21 Apr 2023 17:19:32 +0200 Subject: [PATCH 2/2] Add waits --- src/tests/functional/test_submissions.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/tests/functional/test_submissions.py b/src/tests/functional/test_submissions.py index fd180e1f1..4315b1cd0 100644 --- a/src/tests/functional/test_submissions.py +++ b/src/tests/functional/test_submissions.py @@ -13,21 +13,6 @@ SHORT_WAIT = 0.2 -def retry_test(assertion, max_retries=3): - retries = 0 - success = False - while retries < max_retries and not success: - try: - assert assertion - except AssertionError: - retries += 1 - if retries < max_retries: - print(f"Retrying the test, attempt {retries} of {max_retries}...") - else: - print(f"Test failed after {max_retries} attempts.") - raise - - class TestSubmissions(SeleniumTestCase): def setUp(self): super().setUp() @@ -65,10 +50,10 @@ def _run_submission_and_add_to_leaderboard(self, competition_zip_path, submissio assert self.find_text_in_class('.submission-output-container .title', f"Running {submission_zip_path}", timeout=timeout) # Inside the accordion the output is being streamed - self.wait(SHORT_WAIT) + self.wait(LONG_WAIT) self.find('.submission-output-container .title').click() - # 3 retries, this tests randomly fails, maybe because a problem of synchronization - retry_test(self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout), max_retries=3) + self.wait(LONG_WAIT) + assert self.find_text_in_class('.submission_output', expected_submission_output, timeout=timeout) # The submission table lists our submission! assert self.find('submission-manager#user-submission-table table tbody tr:nth-of-type(1) td:nth-of-type(2)').text == submission_zip_path