From bee93d2f9fd3486954fae10eb23497b156d81939 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 13 Jun 2023 01:18:02 +0500 Subject: [PATCH 01/12] phase status fixed. now used in api and views --- src/apps/api/serializers/competitions.py | 75 +++++++++++++++++++ src/static/riot/competitions/detail/_tabs.tag | 34 --------- .../competitions/detail/submission_upload.tag | 18 ++--- 3 files changed, 83 insertions(+), 44 deletions(-) diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index 0314d831d..69d352c67 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -15,11 +15,13 @@ from tasks.models import Task from api.serializers.queues import QueueSerializer +from datetime import datetime class PhaseSerializer(WritableNestedModelSerializer): tasks = serializers.SlugRelatedField(queryset=Task.objects.all(), required=True, allow_null=False, slug_field='key', many=True) + status = serializers.SerializerMethodField() class Meta: model = Phase @@ -42,6 +44,42 @@ class Meta: 'is_final_phase', ) + def get_status(self, obj): + + now = datetime.now().replace(tzinfo=None) + start = obj.start.replace(tzinfo=None) + end = obj.end.replace(tzinfo=None) if obj.end else obj.end + phase_ended = False + phase_started = False + + # check if phase has started + if start > now: + # start date is in the future, phase started = NO + phase_started = False + else: + # start date is not in the future, phase started = YES + phase_started = True + + if phase_started: + # check if end date exists for this phase + if end: + if end < now: + # Phase cannote accept submissions if end date is in the past + phase_ended = True + else: + # Phase can accept submissions if end date is in the future + phase_ended = False + else: + # Phase can accept submissions if end date is not given + phase_ended = False + + if phase_started and phase_ended: + return Phase.PREVIOUS + elif phase_started and (not phase_ended): + return Phase.CURRENT + elif not phase_started: + return Phase.NEXT + def validate_leaderboard(self, value): if not value: raise ValidationError("Phases require a leaderboard") @@ -50,6 +88,7 @@ def validate_leaderboard(self, value): class PhaseDetailSerializer(serializers.ModelSerializer): tasks = PhaseTaskInstanceSerializer(source='task_instances', many=True) + status = serializers.SerializerMethodField() class Meta: model = Phase @@ -71,6 +110,42 @@ class Meta: 'is_final_phase', ) + def get_status(self, obj): + + now = datetime.now().replace(tzinfo=None) + start = obj.start.replace(tzinfo=None) + end = obj.end.replace(tzinfo=None) if obj.end else obj.end + phase_ended = False + phase_started = False + + # check if phase has started + if start > now: + # start date is in the future, phase started = NO + phase_started = False + else: + # start date is not in the future, phase started = YES + phase_started = True + + if phase_started: + # check if end date exists for this phase + if end: + if end < now: + # Phase cannote accept submissions if end date is in the past + phase_ended = True + else: + # Phase can accept submissions if end date is in the future + phase_ended = False + else: + # Phase can accept submissions if end date is not given + phase_ended = False + + if phase_started and phase_ended: + return Phase.PREVIOUS + elif phase_started and (not phase_ended): + return Phase.CURRENT + elif not phase_started: + return Phase.NEXT + class PhaseUpdateSerializer(PhaseSerializer): tasks = PhaseTaskInstanceSerializer(source='task_instances', many=True) diff --git a/src/static/riot/competitions/detail/_tabs.tag b/src/static/riot/competitions/detail/_tabs.tag index 3ef4279b3..6102bf2cf 100644 --- a/src/static/riot/competitions/detail/_tabs.tag +++ b/src/static/riot/competitions/detail/_tabs.tag @@ -289,40 +289,6 @@ }) }) - // loop over competition phases to mark if phase has started or ended - self.competition.phases.forEach(function (phase, index) { - - phase_ended = false - phase_started = false - - // check if phase has started - if((Date.parse(phase["start"]) - Date.parse(new Date())) > 0){ - // start date is in the future, phase started = NO - phase_started = false - }else{ - // start date is not in the future, phase started = YES - phase_started = true - } - - if(phase_started){ - // check if end data exists for this phase - if(phase["end"]){ - if((Date.parse(phase["end"]) - Date.parse(new Date())) < 0){ - // Phase cannote accept submissions if end date is in the past - phase_ended = true - }else{ - // Phase can accept submissions if end date is in the future - phase_ended = false - } - }else{ - // Phase can accept submissions if end date is not given - phase_ended = false - } - } - self.competition.phases[index]["phase_ended"] = phase_ended - self.competition.phases[index]["phase_started"] = phase_started - }) - self.competition.is_admin = CODALAB.state.user.has_competition_admin_privileges(competition) self.selected_phase_index = _.get(_.find(self.competition.phases, {'status': 'Current'}), 'id') if (self.selected_phase_index == null) { diff --git a/src/static/riot/competitions/detail/submission_upload.tag b/src/static/riot/competitions/detail/submission_upload.tag index 76728a5ee..792842582 100644 --- a/src/static/riot/competitions/detail/submission_upload.tag +++ b/src/static/riot/competitions/detail/submission_upload.tag @@ -1,11 +1,10 @@ -
+

Submission upload

-
This phase has ended and no longer accepts submissions!
-
This phase hasn't started yet!
+
This phase has ended and no longer accepts submissions!
+
This phase hasn't started yet!

Metadata or Fact Sheet

@@ -353,7 +352,7 @@ self.check_can_upload = function () { // Check if selected phase accepts submissions (within the deadline of the phase) - if(self.selected_phase.phase_started && !self.selected_phase.phase_ended){ + if(self.selected_phase.status === "Current"){ CODALAB.api.can_make_submissions(self.selected_phase.id) .done(function (data) { @@ -368,13 +367,12 @@ }) }else{ // Error when phase is not accepting submissions - if(!self.selected_phase.phase_started){ + if(self.selected_phase.status === "Next"){ toastr.error('This phase has not started yet. Please check the phase start date!') - }else { - if(self.selected_phase.phase_ended){ - toastr.error('This phase has ended and no longer accepts submissions!') - } + } + if(self.selected_phase.status === "Previous"){ + toastr.error('This phase has ended and no longer accepts submissions!') } self.clear_form() } From 17c9c09ebb2f452b10baebf61b106cf1ec4a184d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 16 Jun 2023 17:59:34 +0500 Subject: [PATCH 02/12] Resource interface : Cleanup and Quota (#918) * resource interface show orphans and delete option * Flake errors fixed * cleanup delete apis implemented * flake error * resource interface show orphans and delete option * Flake errors fixed * cleanup delete apis implemented * flake error * unused tasks and unused datasets fixed * flake * comments fixed --------- Co-authored-by: didayolo --- src/apps/api/urls.py | 19 +- src/apps/api/views/quota.py | 133 +++++++++++++ src/static/js/ours/client.js | 19 ++ src/static/riot/datasets/management.tag | 6 + src/static/riot/quota_management.tag | 186 ++++++++++++++++++ .../riot/submissions/resource_submissions.tag | 6 + src/static/riot/tasks/management.tag | 6 + src/templates/datasets/management.html | 1 + src/templates/tasks/management.html | 1 + 9 files changed, 376 insertions(+), 1 deletion(-) create mode 100644 src/apps/api/views/quota.py create mode 100644 src/static/riot/quota_management.tag diff --git a/src/apps/api/urls.py b/src/apps/api/urls.py index 2f79d0eff..98957d42b 100644 --- a/src/apps/api/urls.py +++ b/src/apps/api/urls.py @@ -7,7 +7,17 @@ from rest_framework.permissions import AllowAny from rest_framework.urlpatterns import format_suffix_patterns -from .views import analytics, competitions, datasets, profiles, leaderboards, submissions, tasks, queues +from .views import ( + analytics, + competitions, + datasets, + profiles, + leaderboards, + submissions, + tasks, + queues, + quota +) router = SimpleRouter() @@ -45,6 +55,13 @@ path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), path('api-token-auth/', obtain_auth_token), + # User quota and cleanup + path('user_quota_cleanup/', quota.user_quota_cleanup), + path('delete_unused_tasks/', quota.delete_unused_tasks), + path('delete_unused_datasets/', quota.delete_unused_datasets), + path('delete_unused_submissions/', quota.delete_unused_submissions), + path('delete_failed_submissions/', quota.delete_failed_submissions), + # API Docs re_path(r'docs(?P\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), path('docs/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), diff --git a/src/apps/api/views/quota.py b/src/apps/api/views/quota.py new file mode 100644 index 000000000..7be296639 --- /dev/null +++ b/src/apps/api/views/quota.py @@ -0,0 +1,133 @@ +from django.db.models import Q +from rest_framework.decorators import api_view +from rest_framework.response import Response +from datasets.models import Data +from tasks.models import Task +from competitions.models import Submission + + +@api_view(['GET']) +def user_quota_cleanup(request): + + # Get Unused tasks count + unused_tasks = Task.objects.filter( + created_by=request.user, + phases__isnull=True + ).count() + + # Get Unused datasets and programs count + unused_datasets_programs = Data.objects.filter( + Q(created_by=request.user) & + ~Q(type=Data.SUBMISSION) & + ~Q(type=Data.COMPETITION_BUNDLE) + ).exclude( + Q(task_ingestion_programs__isnull=False) | + Q(task_input_datas__isnull=False) | + Q(task_reference_datas__isnull=False) | + Q(task_scoring_programs__isnull=False) + ).count() + + # Get Unused submissions count + unused_submissions = Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.SUBMISSION) & + Q(competition__isnull=True) + ).count() + + # Get Failed submissions count + failed_submissions = Submission.objects.filter( + Q(owner=request.user) & + Q(status=Submission.FAILED) + ).count() + + return Response({ + "unused_tasks": unused_tasks, + "unused_datasets_programs": unused_datasets_programs, + "unused_submissions": unused_submissions, + "failed_submissions": failed_submissions + }) + + +@api_view(['DELETE']) +def delete_unused_tasks(request): + try: + + Task.objects.filter( + created_by=request.user, + phases__isnull=True + ).delete() + + return Response({ + "success": True, + "message": "Unused tasks deleted successfully" + }) + except Exception as e: + return Response({ + "success": False, + "message": e + }) + + +@api_view(['DELETE']) +def delete_unused_datasets(request): + try: + Data.objects.filter( + Q(created_by=request.user) & + ~Q(type=Data.SUBMISSION) & + ~Q(type=Data.COMPETITION_BUNDLE) + ).exclude( + Q(task_ingestion_programs__isnull=False) | + Q(task_input_datas__isnull=False) | + Q(task_reference_datas__isnull=False) | + Q(task_scoring_programs__isnull=False) + ).delete() + + return Response({ + "success": True, + "message": "Unused datasets and programs deleted successfully" + }) + except Exception as e: + return Response({ + "success": False, + "message": e + }) + + +@api_view(['DELETE']) +def delete_unused_submissions(request): + try: + + Data.objects.filter( + Q(created_by=request.user) & + Q(type=Data.SUBMISSION) & + Q(competition__isnull=True) + ).delete() + + return Response({ + "success": True, + "message": "Unused submissions deleted successfully" + }) + except Exception as e: + return Response({ + "success": False, + "message": e + }) + + +@api_view(['DELETE']) +def delete_failed_submissions(request): + try: + Submission.objects.filter( + Q(owner=request.user) & + Q(status=Submission.FAILED) + ).delete() + + return Response({ + "success": True, + "message": "Failed submissions deleted successfully" + }) + except Exception as e: + return Response({ + "success": False, + "message": e + }) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 5a912ffb4..3c4a6ae05 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -294,4 +294,23 @@ CODALAB.api = { get_analytics: (filters) => { return CODALAB.api.request('GET', `${URLS.API}analytics/`, filters) }, + /*--------------------------------------------------------------------- + User Quota and Cleanup + ---------------------------------------------------------------------*/ + get_user_quota_cleanup: () => { + return CODALAB.api.request('GET', `${URLS.API}user_quota_cleanup/`) + }, + delete_unused_tasks: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_tasks/`) + }, + delete_unused_datasets: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_datasets/`) + }, + delete_unused_submissions: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_unused_submissions/`) + }, + delete_failed_submissions: () => { + return CODALAB.api.request('DELETE', `${URLS.API}delete_failed_submissions/`) + }, + } diff --git a/src/static/riot/datasets/management.tag b/src/static/riot/datasets/management.tag index eb6a257ed..7b782472c 100644 --- a/src/static/riot/datasets/management.tag +++ b/src/static/riot/datasets/management.tag @@ -305,6 +305,7 @@ .done(function () { self.update_datasets() toastr.success("Dataset deleted successfully!") + CODALAB.events.trigger('reload_quota_cleanup') }) .fail(function (response) { toastr.error(response.responseJSON['error']) @@ -320,6 +321,7 @@ self.update_datasets() toastr.success("Dataset deleted successfully!") self.marked_datasets = [] + CODALAB.events.trigger('reload_quota_cleanup') }) .fail(function (response) { for (e in response.responseJSON) { @@ -396,6 +398,7 @@ self.update_datasets() self.clear_form() $(self.refs.dataset_creation_modal).modal('hide') + CODALAB.events.trigger('reload_quota_cleanup') }) .fail(function (response) { if (response) { @@ -469,6 +472,9 @@ return(n.toFixed(1) + ' ' + units[i]); } + // Update datasets on unused datasets delete + CODALAB.events.on('reload_datasets', self.update_datasets) + From 5205b0a4453b74bb0cd73b035f2900a55c6ec01d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 22 Jun 2023 18:59:02 +0500 Subject: [PATCH 10/12] Organization delete functionality added (#959) * Organization delete functionality added * Codalab->Codabench * delte restricted to owner only --- src/apps/api/views/profiles.py | 22 ++++++++++++ src/static/js/ours/client.js | 3 ++ .../profiles/organization_detail.html | 36 ++++++++++++++++++- 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/apps/api/views/profiles.py b/src/apps/api/views/profiles.py index c34fb9482..6492a5efb 100644 --- a/src/apps/api/views/profiles.py +++ b/src/apps/api/views/profiles.py @@ -229,3 +229,25 @@ def validate_invite(self, request): mem_ser = MembershipSerializer(membership) return Response(mem_ser.data, status=status.HTTP_200_OK) + + @action(detail=True, methods=['delete']) + def delete_organization(self, request, pk=None): + try: + org = Organization.objects.get(id=pk) + member = org.membership_set.get(user=request.user) + if member.group == Membership.OWNER: + org.delete() + return Response({ + "success": True, + "message": "Organization deleted!" + }) + else: + return Response({ + "success": False, + "message": "You do not have delete rights!" + }) + except Exception as e: + return Response({ + "success": False, + "message": f"{e}" + }) diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 3c4a6ae05..f3e1a21dc 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -276,6 +276,9 @@ CODALAB.api = { delete_organization_member: (id, data) => { return CODALAB.api.request('DELETE', `${URLS.API}organizations/${id}/delete_member/`, data) }, + delete_organization: (id) => { + return CODALAB.api.request('DELETE', `${URLS.API}organizations/${id}/delete_organization/`) + }, /*--------------------------------------------------------------------- Participants ---------------------------------------------------------------------*/ diff --git a/src/templates/profiles/organization_detail.html b/src/templates/profiles/organization_detail.html index 8b99319a1..386403a22 100644 --- a/src/templates/profiles/organization_detail.html +++ b/src/templates/profiles/organization_detail.html @@ -6,6 +6,18 @@ {% if is_editor %}
+ + {% if organization.users|length > 1 %} + + {% else %} + + {% endif %} + +
{% endif %}
@@ -29,7 +41,7 @@

{{ organization.name }}

{{ organization.description | linebreaks }}

- Participating in Codalab since {{ organization.date_created }} + Participating in Codabench since {{ organization.date_created }}
@@ -77,3 +89,25 @@

Users

{% endblock %} + +{% block extra_js %} + +{% endblock %} From 7b72adf3cb5724272f0fa5de1b5ff3371c10237e Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 23 Jun 2023 16:19:21 +0500 Subject: [PATCH 11/12] adding new phase in competition fixed (#963) * adding new phase in competition fixed * unused import removed * auto_migrate_removed from new created phase * transaction added * no end date handled --- src/apps/api/views/competitions.py | 84 +++++++++++++++++++----------- 1 file changed, 54 insertions(+), 30 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index d595338a3..2db303a45 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -36,8 +36,8 @@ from leaderboards.models import Leaderboard from utils.data import make_url_sassy from api.permissions import IsOrganizerOrCollaborator -import logging -logger = logging.getLogger() +from datetime import datetime +from django.db import transaction class CompetitionViewSet(ModelViewSet): @@ -185,34 +185,58 @@ def create(self, request, *args, **kwargs): return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) def update(self, request, *args, **kwargs): - """Mostly a copy of the underlying base update, however we return some additional data - in the response to remove a GET from the frontend""" - partial = kwargs.pop('partial', False) - instance = self.get_object() - data = request.data - # TODO - This is Temporary. Need to change Leaderboard to Phase connect to M2M and handle this correctly. - # save leaderboard individually, then pass pk to each phase - if 'leaderboards' in data: - leaderboard_data = data['leaderboards'][0] - if(leaderboard_data['id']): - leaderboard_instance = Leaderboard.objects.get(id=leaderboard_data['id']) - leaderboard = LeaderboardSerializer(leaderboard_instance, data=data['leaderboards'][0]) - else: - leaderboard = LeaderboardSerializer(data=data['leaderboards'][0]) - leaderboard.is_valid() - leaderboard.save() - leaderboard_id = leaderboard["id"].value - for phase in data['phases']: - phase['leaderboard'] = leaderboard_id - - serializer = self.get_serializer(instance, data=data, partial=partial) - serializer.is_valid(raise_exception=True) - self.perform_update(serializer) - - if getattr(instance, '_prefetched_objects_cache', None): - # If 'prefetch_related' has been applied to a queryset, we need to - # forcibly invalidate the prefetch cache on the instance. - instance._prefetched_objects_cache = {} + # Execute everything inside atomic transaction + with transaction.atomic(): + """Mostly a copy of the underlying base update, however we return some additional data + in the response to remove a GET from the frontend""" + partial = kwargs.pop('partial', False) + instance = self.get_object() + data = request.data + # TODO - This is Temporary. Need to change Leaderboard to Phase connect to M2M and handle this correctly. + # save leaderboard individually, then pass pk to each phase + if 'leaderboards' in data: + leaderboard_data = data['leaderboards'][0] + if(leaderboard_data['id']): + leaderboard_instance = Leaderboard.objects.get(id=leaderboard_data['id']) + leaderboard = LeaderboardSerializer(leaderboard_instance, data=data['leaderboards'][0]) + else: + leaderboard = LeaderboardSerializer(data=data['leaderboards'][0]) + leaderboard.is_valid() + leaderboard.save() + leaderboard_id = leaderboard["id"].value + + for phase in data['phases']: + # Newly added phase from front-end has no id + # Add a phase first to get id + # add this phase id in each task + if 'id' not in phase: + # Create Phase object + new_phase_obj = Phase.objects.create( + status=phase["status"], + index=phase["index"], + start=datetime.strptime(phase['start'], "%B %d, %Y"), + end=datetime.strptime(phase['end'], "%B %d, %Y") if phase['end'] else None, + name=phase["name"], + description=phase["description"], + hide_output=phase["hide_output"], + competition=Competition.objects.get(id=data['id']) + ) + # Get phase id + new_phase_id = new_phase_obj.id + # loop over phase tasks to add phase id in each task + for task in phase["tasks"]: + task['phase'] = new_phase_id + + phase['leaderboard'] = leaderboard_id + + serializer = self.get_serializer(instance, data=data, partial=partial) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + if getattr(instance, '_prefetched_objects_cache', None): + # If 'prefetch_related' has been applied to a queryset, we need to + # forcibly invalidate the prefetch cache on the instance. + instance._prefetched_objects_cache = {} # Re-do serializer in detail version (i.e. for Collaborator data) context = self.get_serializer_context() From 469cdd91f765c22ab3e231ea00ce6583ce5e9e4e Mon Sep 17 00:00:00 2001 From: didayolo Date: Fri, 23 Jun 2023 13:47:14 +0200 Subject: [PATCH 12/12] Update text in home.html --- src/templates/pages/home.html | 107 +++++++++++++--------------------- 1 file changed, 39 insertions(+), 68 deletions(-) diff --git a/src/templates/pages/home.html b/src/templates/pages/home.html index fb586f5a4..8ebde129d 100644 --- a/src/templates/pages/home.html +++ b/src/templates/pages/home.html @@ -88,111 +88,82 @@

- About CodaBench + About Codabench
-

What is CodaBench?

+

What is Codabench?

-

Codabench is a platform allowing you to flexibly specify a benchmark. First you define tasks, e.g. datasets and metrics of success, then you specify the API for submissions of code (algorithms), add some documentation pages, and [CLICK] your benchmark is created, ready to accept submissions of new algorithms. Participant results get appended to an ever-growing leaderboard.

-

You may also create inverted benchmarks in which the role of datasets and algorithms are swapped. You specify reference algorithms and your participants submit datasets.

+

Codabench is an open source platform allowing you to organize AI benchmarks. It is flexible and powerful, yet easy to use. You define tasks (e.g. datasets and metrics of success), then interface for submissions of code (algorithms), add some documentation pages, + make an upload and that's it! Your benchmark is created, ready to accept submissions of new algorithms. Everything can be fully customized, including the code of the scoring program. Organizers can even hook up their own compute workers to their benchmarks, enabling unlimited computing power. + Participants can try out their methods, get real-time feedback and results on a competitive leaderboard, detailed plots and more.

+

In a unique twist, Codabench also allows you to create inverted benchmarks. Here, the roles of datasets and algorithms are interchanged. In this scenario, you set the reference algorithms and the participants contribute datasets.

-

What is Codalab?

+

What is CodaLab?

-

CodaLab Competitions is a powerful open source framework for running competitions that - involve - result or code submission. You can either participate in an existing competition or host - a new - competition.

-

Most competitions hosted on Codalab are machine learning (data science) - competitions, but Codalab is NOT limited to this application domain. It can accommodate - any - problem for which a solution can be provided in the form of a zip archive containing a - number of - files to be evaluated quantitatively by a scoring program (provided by the organizers). - The - scoring program must return a numeric score, which is displayed on a leaderboard where - the - performances of participants are compared.

+

CodaLab Competitions is a powerful open source framework for running competitions using result or code submissions. + You can participate in an existing competition or host your own competition for free.

+

Most competitions hosted on CodaLab are machine learning (data science) competitions, but it is NOT limited to this application domain. + It can accommodate any problem for which a solution can be provided in the form of a zip archive containing a + number of files to be evaluated quantitatively by a scoring program (provided by the organizers). + The scoring program must return a numeric score, which is displayed on a leaderboard where + the performances of participants are compared.

-

History of Codalab

+

History of CodaLab

-

Codalab was created in 2013 as a joint venture between Microsoft and Stanford University. +

CodaLab was created in 2013 as a joint venture between Microsoft and Stanford University. Originally the vision was to create an ecosystem for conducting computational research - in a more - efficient, reproducible, and collaborative manner, combining worksheets and - competitions. - Worksheets capture complex research pipelines in a reproducible way and create - "executable - papers". Currently, we are developing the V2 of Codalab, which will be able to organize benchmarks.

+ in a more efficient, reproducible, and collaborative manner, combining worksheets and + competitions. Worksheets capture complex research pipelines in a reproducible way and create "executable + papers". Codabench is the continuity of CodaLab, a version 2 in which users can organize benchmarks.

Some competitions have been organized using worksheets, but the competition platform and the worksheet platform have both a large user base and can be used independently. In 2014, ChaLearn joined to co-develop - Codalab - competitions. Since - 2015, University Paris-Saclay is - community lead of Codalab competitions, under the direction of Isabelle Guyon, professor - of big - data. Codalab is administered by CKCollab and the LRI - staff. + CodaLab competitions. Since 2015, University Paris-Saclay is + community lead of CodaLab competitions, under the direction of Isabelle Guyon, professor + of big data. CodaLab and Codabench are administered by the LISN staff.

-

Codalab in Research

+

CodaLab in Research

-

Codalab is used actively in research. In +

CodaLab is used actively in research. In 2019/2020, 400 new challenges were launched. Recent - popular challenges organized with Codalab include the - COVID-19 + popular challenges organized with CodaLab include the + COVID-19 retweet prediction challenge,  the ECCV 2020 ChaLearn LAP Fair face recognition challenge, - the 2020 DriveML + the 2020 DriveML Huawei Autonomous Vehicle Challenge, and high - profile challenges include the 2 - million Euro prize of the EU, organized by the See.4C - consortium, the 2 + million Euro prize of the EU, organized by the See.4C consortium, the CIKM - AnalytiCup 2017, which attracted 493 participants, MSCOCO (633 participants) and the ChaLearn - AutoML challenge 2017 (687 participants).

- -

Since 2016, Codalab offers the possibility of organizing machine learning challenges with - code - submission. The simplest machine learning challenges require only the submission of - results, - which are compared to a solution (or key) by a scoring program. Result submission - challenges are - less computationally expensive than code submission challenges. However, they offer less +

Since 2016, CodaLab offers the possibility of organizing machine learning challenges with + code submission. The simplest machine learning challenges require only the submission of + results, which are compared to a solution (or key) by a scoring program. Result submission + challenges are less computationally expensive than code submission challenges. However, they offer less possibilities. In particular, code submission allows conducting fair benchmarks by - executing - submitted code in the same condition for all participants.

-

Codalab has been providing free resources for challenge organizers who want to run high - impact - events, within a pre-approved agreed upon budget. New since version 1.5: organizers can - hook up - their own compute workers to the backend of Codalab to redirect the code submissions, - enabling - growth to big data competitions running at the expense of the organizers. For very - special - dedicated projects, Codalab can be customized since it is an open source project.

+ executing submitted code in the same condition for all participants.

+

CodaLab has been providing free resources for challenge organizers who want to run high + impact events, within a pre-approved agreed upon budget. New since version 1.5: organizers can + hook up their own compute workers to the backend of CodaLab to redirect the code submissions, + enabling growth to big data competitions running at the expense of the organizers. For very + special dedicated projects, CodaLab can be customized since it is an open source project.