diff --git a/bin/pg_dump.py b/bin/pg_dump.py index 88f93d548..9d266a6d3 100755 --- a/bin/pg_dump.py +++ b/bin/pg_dump.py @@ -17,7 +17,7 @@ call([ 'docker', 'exec', - 'codabench-db-1', + 'db', 'bash', '-c', f'PGPASSWORD=$DB_PASSWORD pg_dump -Fc -U $DB_USERNAME $DB_NAME > /app/backups/{dump_name}' @@ -25,5 +25,5 @@ # Push/destroy dump call([ - 'docker', 'exec', 'codabench-django-1', 'python', 'manage.py', 'upload_backup', f'{dump_name}' + 'docker', 'exec', 'django', 'python', 'manage.py', 'upload_backup', f'{dump_name}' ]) diff --git a/pyproject.toml b/pyproject.toml index 9662fa05c..c007d2d88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ dev = [ "django-querycount==0.7.0", "django-debug-toolbar==6.2.0", "flake8==7.3.0", - "pytest==9.0.2", + "pytest==9.0.3", "pytest-django==4.12.0", ] [tool.pytest.ini_options] diff --git a/src/apps/api/tests/test_cleanup.py b/src/apps/api/tests/test_cleanup.py index a927800af..1dd67a2ce 100644 --- a/src/apps/api/tests/test_cleanup.py +++ b/src/apps/api/tests/test_cleanup.py @@ -56,8 +56,19 @@ def setUp(self): DataFactory(created_by=user, type=Data.INGESTION_PROGRAM), DataFactory(created_by=user, type=Data.SCORING_PROGRAM), DataFactory(created_by=user, type=Data.INPUT_DATA), - DataFactory(created_by=user, type=Data.REFERENCE_DATA), - DataFactory(created_by=user, type=Data.PUBLIC_DATA) + DataFactory(created_by=user, type=Data.REFERENCE_DATA) + ] + + # Create unused starting kits + self.unused_starting_kits = [ + DataFactory(created_by=user, type=Data.STARTING_KIT), + DataFactory(created_by=user, type=Data.STARTING_KIT) + ] + + # Create unused competition bundles + self.unused_competition_bundles = [ + DataFactory(created_by=user, type=Data.COMPETITION_BUNDLE), + DataFactory(created_by=user, type=Data.COMPETITION_BUNDLE) ] self.client.login(username='test_user', password='test_user') @@ -72,6 +83,8 @@ def test_cleanup_stats(self): assert content["unused_datasets_programs"] == len(self.unused_datasets_programs) assert content["unused_submissions"] == len(self.unused_submissions) assert content["failed_submissions"] == len(self.failed_submissions) + assert content["unused_starting_kits"] == len(self.unused_starting_kits) + assert content["unused_competition_bundles"] == len(self.unused_competition_bundles) def test_delete_unused_tasks(self): @@ -132,3 +145,33 @@ def test_delete_failed_submissions(self): assert resp.status_code == 200 content = json.loads(resp.content) assert content["failed_submissions"] == 0 + + def test_delete_unused_starting_kits(self): + + url = reverse('delete_unused_starting_kits') + resp = self.client.delete(url) + assert resp.status_code == 200 + content = json.loads(resp.content) + assert content["success"] + assert content["message"] == "Unused starting kits deleted successfully" + + url = reverse('user_quota_cleanup') + resp = self.client.get(url) + assert resp.status_code == 200 + content = json.loads(resp.content) + assert content["unused_starting_kits"] == 0 + + def test_delete_unused_competition_bundles(self): + + url = reverse('delete_unused_competition_bundles') + resp = self.client.delete(url) + assert resp.status_code == 200 + content = json.loads(resp.content) + assert content["success"] + assert content["message"] == "Unused competition bundles deleted successfully" + + url = reverse('user_quota_cleanup') + resp = self.client.get(url) + assert resp.status_code == 200 + content = json.loads(resp.content) + assert content["unused_competition_bundles"] == 0 diff --git a/src/apps/api/urls.py b/src/apps/api/urls.py index cad6fde62..640b8a954 100644 --- a/src/apps/api/urls.py +++ b/src/apps/api/urls.py @@ -54,6 +54,8 @@ path('delete_unused_datasets/', quota.delete_unused_datasets, name="delete_unused_datasets"), path('delete_unused_submissions/', quota.delete_unused_submissions, name="delete_unused_submissions"), path('delete_failed_submissions/', quota.delete_failed_submissions, name="delete_failed_submissions"), + path('delete_unused_starting_kits/', quota.delete_unused_starting_kits, name="delete_unused_starting_kits"), + path('delete_unused_competition_bundles/', quota.delete_unused_competition_bundles, name="delete_unused_competition_bundles"), # User account path('delete_account/', profiles.delete_account, name="delete_account"), diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 7cacb433b..53b9d2193 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -18,7 +18,7 @@ from rest_framework.response import Response from rest_framework.renderers import JSONRenderer from rest_framework_csv.renderers import CSVRenderer -from api.pagination import LargePagination +from api.pagination import DynamicChoicePagination, LargePagination from api.renderers import ZipRenderer from rest_framework.viewsets import ModelViewSet from api.serializers.competitions import CompetitionSerializerSimple, PhaseSerializer, \ @@ -774,10 +774,17 @@ def rerun_submissions(self, request, pk): def get_leaderboard(self, request, pk): phase = self.get_object() if phase.competition.fact_sheet: - fact_sheet_keys = [(phase.competition.fact_sheet[question]['key'], phase.competition.fact_sheet[question]['title']) - for question in phase.competition.fact_sheet if phase.competition.fact_sheet[question]['is_on_leaderboard'] == 'true'] + fact_sheet_keys = [ + ( + phase.competition.fact_sheet[question]['key'], + phase.competition.fact_sheet[question]['title'] + ) + for question in phase.competition.fact_sheet + if phase.competition.fact_sheet[question]['is_on_leaderboard'] == 'true' + ] else: fact_sheet_keys = None + query = LeaderboardPhaseSerializer(phase).data response = { 'title': query['leaderboard']['title'], @@ -787,9 +794,11 @@ def get_leaderboard(self, request, pk): 'fact_sheet_keys': fact_sheet_keys or None, 'primary_index': query['leaderboard']['primary_index'] } + columns = [col for col in query['columns']] submissions_keys = {} submission_detailed_results = {} + for submission in query['submissions']: submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}" # gather detailed result from submissions for each task @@ -814,6 +823,7 @@ def get_leaderboard(self, request, pk): 'organization': submission['organization'], 'created_when': submission['created_when'] }) + for score in submission['scores']: # to check if a column is found @@ -851,6 +861,21 @@ def get_leaderboard(self, request, pk): for k, v in submissions_keys.items(): response['submissions'][v]['detailed_results'] = submission_detailed_results[k] + # --- pagination addition --- + total_count = len(response['submissions']) + paginator = DynamicChoicePagination() + paginated_submissions = paginator.paginate_queryset(response['submissions'], request, view=self) + if paginated_submissions is None: + paginated_submissions = response['submissions'] + + response['submissions'] = paginated_submissions + response['count'] = total_count + response['page_size'] = getattr(paginator, 'requested_page_size', request.query_params.get('page_size', 50)) + response['next'] = paginator.get_next_link() + response['previous'] = paginator.get_previous_link() + response['allowed_page_sizes'] = [50, 100, 500, 'all'] + # --- end pagination addition --- + for task in query['tasks']: # This can be used to rendered variable columns on each task tempTask = { @@ -862,6 +887,7 @@ def get_leaderboard(self, request, pk): for col in columns: tempTask['columns'].append(col) response['tasks'].append(tempTask) + return Response(response) diff --git a/src/apps/api/views/quota.py b/src/apps/api/views/quota.py index 0b99ff3a2..6cc927517 100644 --- a/src/apps/api/views/quota.py +++ b/src/apps/api/views/quota.py @@ -21,7 +21,9 @@ def user_quota_cleanup(request): unused_datasets_programs = Data.objects.filter( Q(created_by=request.user) & ~Q(type=Data.SUBMISSION) & - ~Q(type=Data.COMPETITION_BUNDLE) + ~Q(type=Data.COMPETITION_BUNDLE) & + ~Q(type=Data.PUBLIC_DATA) & + ~Q(type=Data.STARTING_KIT) ).exclude( Q(task_ingestion_programs__isnull=False) | Q(task_input_datas__isnull=False) | @@ -42,11 +44,29 @@ def user_quota_cleanup(request): Q(status=Submission.FAILED) ).count() + # Get unused starting kits count + unused_starting_kits = Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.STARTING_KIT) & + Q(competition__isnull=True) & + Q(phase_starting_kit__isnull=True) + ).count() + + # Get unused competition bundles + unused_competition_bundles = Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.COMPETITION_BUNDLE) & + Q(competition__isnull=True) & + Q(competition_bundles__isnull=True) + ).count() + return Response({ "unused_tasks": unused_tasks, "unused_datasets_programs": unused_datasets_programs, "unused_submissions": unused_submissions, - "failed_submissions": failed_submissions + "failed_submissions": failed_submissions, + "unused_starting_kits": unused_starting_kits, + "unused_competition_bundles": unused_competition_bundles }) @@ -84,7 +104,9 @@ def delete_unused_datasets(request): Data.objects.filter( Q(created_by=request.user) & ~Q(type=Data.SUBMISSION) & - ~Q(type=Data.COMPETITION_BUNDLE) + ~Q(type=Data.COMPETITION_BUNDLE) & + ~Q(type=Data.PUBLIC_DATA) & + ~Q(type=Data.STARTING_KIT) ).exclude( Q(task_ingestion_programs__isnull=False) | Q(task_input_datas__isnull=False) | @@ -144,3 +166,47 @@ def delete_failed_submissions(request): "success": False, "message": f"{e}" }) + + +@api_view(['DELETE']) +def delete_unused_starting_kits(request): + try: + Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.STARTING_KIT) & + Q(competition__isnull=True) & + Q(phase_starting_kit__isnull=True) + ).delete() + + return Response({ + "success": True, + "message": "Unused starting kits deleted successfully" + }) + except Exception as e: + logger.error(f"UNUSED STARTING KITS DELETION --- {e}") + return Response({ + "success": False, + "message": f"{e}" + }) + + +@api_view(['DELETE']) +def delete_unused_competition_bundles(request): + try: + Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.COMPETITION_BUNDLE) & + Q(competition__isnull=True) & + Q(competition_bundles__isnull=True) + ).delete() + + return Response({ + "success": True, + "message": "Unused competition bundles deleted successfully" + }) + except Exception as e: + logger.error(f"UNUSED COMPETITION BUNDLES DELETION --- {e}") + return Response({ + "success": False, + "message": f"{e}" + }) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index f43bfaaa2..fa169c5a8 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -147,6 +147,10 @@ CODALAB.api = { get_leaderboard_for_render: function (phase_pk) { return CODALAB.api.request('GET', `${URLS.API}phases/${phase_pk}/get_leaderboard/`) }, + get_leaderboard_for_render: function (phase_pk, params = {}) { + return CODALAB.api.request('GET', `${URLS.API}phases/${phase_pk}/get_leaderboard/`, params) + }, + update_submission_score: function (pk, data) { return CODALAB.api.request('PATCH', `${URLS.API}submission_scores/${pk}/`, data) }, @@ -388,6 +392,12 @@ CODALAB.api = { delete_failed_submissions: () => { return CODALAB.api.request('DELETE', `${URLS.API}delete_failed_submissions/`) }, + delete_unused_starting_kits: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_starting_kits/`) + }, + delete_unused_competition_bundles: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_competition_bundles/`) + }, /*--------------------------------------------------------------------- User Account ---------------------------------------------------------------------*/ diff --git a/src/static/riot/competitions/bundle_management.tag b/src/static/riot/competitions/bundle_management.tag index 635bc9f99..b7253dc5a 100644 --- a/src/static/riot/competitions/bundle_management.tag +++ b/src/static/riot/competitions/bundle_management.tag @@ -4,13 +4,13 @@ - - +
@@ -23,32 +23,32 @@ - - + onclick="{show_info_modal.bind(this, bundle)}"> + - - + + - + @@ -56,8 +56,8 @@ - - - - - - - - - - - - + + + + + + + + + + + +
File Name
{ dataset.name }{ bundle.name } - { pretty_bytes(dataset.file_size) }{ timeSince(Date.parse(dataset.created_when)) } ago{ pretty_bytes(bundle.file_size) }{ timeSince(Date.parse(bundle.created_when)) } ago - -
- +
+
- No Datasets Yet! + No Competition Bundles Yet!
- +
- No submissions have been added to this leaderboard yet! -
- - - - - - {index + 1} - { submission.owner }{ submission.organization.name } - { pretty_date(submission.created_when) } - {submission.id} - - - - {get_score(column, submission)} -
+ No submissions have been added to this leaderboard yet! +
+ + + + + + {get_row_number(index)} + { submission.owner }{ submission.organization.name } + { pretty_date(submission.created_when) } + {submission.id} + + + + {get_score(column, submission)} +
+ + +