diff --git a/circle.yml b/.circleci/config.yml similarity index 97% rename from circle.yml rename to .circleci/config.yml index 78be88b4f..14ce6ddd0 100644 --- a/circle.yml +++ b/.circleci/config.yml @@ -3,7 +3,7 @@ version: 2 jobs: test: machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:2022.07.1 steps: - checkout diff --git a/.env_sample b/.env_sample index ca451e69f..4acfc4528 100644 --- a/.env_sample +++ b/.env_sample @@ -8,7 +8,7 @@ DB_PORT=5432 DJANGO_SETTINGS_MODULE=settings.develop ALLOWED_HOSTS=localhost,example.com -SUBMISSIONS_API_URL=http://example.com/api +SUBMISSIONS_API_URL=http://localhost/api # Local domain definition DOMAIN_NAME=localhost:80 @@ -55,7 +55,7 @@ AWS_SECRET_ACCESS_KEY=testsecret AWS_STORAGE_BUCKET_NAME=public AWS_STORAGE_PRIVATE_BUCKET_NAME=private # NOTE! port 9000 here should match $MINIO_PORT -AWS_S3_ENDPOINT_URL=http://docker.for.mac.localhost:9000/ +AWS_S3_ENDPOINT_URL=http://localhost:9000/ AWS_QUERYSTRING_AUTH=False # # S3 storage example diff --git a/LICENSE b/LICENSE.TXT similarity index 100% rename from LICENSE rename to LICENSE.TXT diff --git a/Procfile b/Procfile index c4dd8e068..f40a1a035 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: cd src && gunicorn asgi:application -w 4 -k uvicorn.workers.UvicornWorker -b :$PORT --max-requests 1024 --max-requests-jitter 256 +web: cd src && gunicorn asgi:application -w 3 -k uvicorn.workers.UvicornWorker -b :$PORT --max-requests 1024 --max-requests-jitter 256 worker: cd src && celery -A celery_config worker -B -Q site-worker -l info -n site-worker@%n --concurrency=3 diff --git a/README.md b/README.md index 7a9d6f485..424098c21 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,21 @@ -# Codabench +![CodaBench logo](src/static/img/codabench_black.png) [![Circle CI](https://circleci.com/gh/codalab/codabench.svg?style=shield)](https://app.circleci.com/pipelines/github/codalab/codabench) -## Installation +## What is CodaBench? + +CodaBench is an open-source web-based platform that enables researchers, developers, and data scientists to collaborate, with the goal of advancing research fields where machine learning and advanced computation is used. CodaBench helps to solve many common problems in the arena of data-oriented research through its online community where people can share worksheets and participate in competitions and benchmarks. It can be seen as a version 2 of [CodaLab Competitions](https://github.com/codalab/codalab-competitions). + +To see CodaBench in action, visit [codabench.org](https://www.codabench.org/). + + +## Documentation + +- [CodaBench Wiki](https://github.com/codalab/codabench/wiki) + + +## Quick installation (for Linux) + +_To participate, or even organize your own benchmarks or competitions, **you don't need to install anything**, you just need to sign in an instance of the platform (e.g. [this one](https://www.codabench.org/)). +If you wish to configure your own instance of CodaBench platform, here are the instructions:_ ``` @@ -15,6 +30,34 @@ You can now login as username "admin" with password "admin" at http://localhost: If you ever need to reset the database, use the script `./reset_db.sh` + +## License + +Copyright (c) 2020-2022, Université Paris-Saclay. +This software is released under the Apache License 2.0 (the "License"); you may not use the software except in compliance with the License. + +The text of the Apache License 2.0 can be found online at: +http://www.opensource.org/licenses/apache2.0.php + + +## Cite CodaBench in your research + +``` +@article{codabench, + title = {Codabench: Flexible, easy-to-use, and reproducible meta-benchmark platform}, + author = {Zhen Xu and Sergio Escalera and Adrien Pavão and Magali Richard and + Wei-Wei Tu and Quanming Yao and Huan Zhao and Isabelle Guyon}, + journal = {Patterns}, + volume = {3}, + number = {7}, + pages = {100543}, + year = {2022}, + issn = {2666-3899}, + doi = {https://doi.org/10.1016/j.patter.2022.100543}, + url = {https://www.sciencedirect.com/science/article/pii/S2666389922001465} +} +``` + ## Running tests ``` @@ -177,3 +220,4 @@ You can execute commands against a role: ``` See available commands with `fab -l` + diff --git a/bin/pg_dump.py b/bin/pg_dump.py index ae171bf65..f8b29c254 100755 --- a/bin/pg_dump.py +++ b/bin/pg_dump.py @@ -2,7 +2,7 @@ """ Usage, in `crontab -e`: - @daily /home/ubuntu/competitions-v2/bin/pg_dump.py + @daily /home/ubuntu/codabench/bin/pg_dump.py """ import time @@ -17,13 +17,15 @@ call([ 'docker', 'exec', - 'competitions-v2_db_1', + 'codabench-db-1', 'bash', '-c', - f'PGPASSWORD=$DB_PASSWORD pg_dump -Fc -U $DB_USER $DB_NAME > /app/backups/{dump_name}' + f'PGPASSWORD=$DB_PASSWORD pg_dump -Fc -U $DB_USERNAME $DB_NAME > /app/backups/{dump_name}' ]) # Push/destroy dump call([ - 'docker', 'exec', 'competitions-v2_django_1', 'python', 'manage.py', 'upload_backup', f'{dump_name}' + 'docker', 'exec', 'codabench-django-1', 'python', 'manage.py', 'upload_backup', f'{dump_name}' ]) + + diff --git a/docker/compute_worker/compute_worker.py b/docker/compute_worker/compute_worker.py index 67a9b1b17..8f13a5a85 100644 --- a/docker/compute_worker/compute_worker.py +++ b/docker/compute_worker/compute_worker.py @@ -261,7 +261,7 @@ def get_detailed_results_file_path(self): async def send_detailed_results(self, file_path): logger.info(f"Updating detailed results {file_path} - {self.detailed_results_url}") - self._put_file(self.detailed_results_url, file=file_path, content_type='') + self._put_file(self.detailed_results_url, file=file_path, content_type='text/html') async with websockets.connect(self.websocket_url) as websocket: await websocket.send(json.dumps({ "kind": 'detailed_result_update', @@ -734,7 +734,7 @@ def push_scores(self): elif os.path.exists(os.path.join(self.output_dir, "scores.txt")): scores_file = os.path.join(self.output_dir, "scores.txt") with open(scores_file) as f: - scores = yaml.load(f) + scores = yaml.load(f, yaml.Loader) else: raise SubmissionException("Could not find scores file, did the scoring program output it?") diff --git a/src/apps/api/serializers/leaderboards.py b/src/apps/api/serializers/leaderboards.py index 5287e5fdf..444425861 100644 --- a/src/apps/api/serializers/leaderboards.py +++ b/src/apps/api/serializers/leaderboards.py @@ -128,7 +128,7 @@ def get_columns(self, instance): if len(columns) == 0: raise serializers.ValidationError("No columns exist on the leaderboard") else: - return ColumnSerializer(columns, many=len(columns) > 1).data + return ColumnSerializer(columns, many=len(columns) >= 1).data class Meta: model = Phase diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 35233ae7e..1a7405151 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -82,6 +82,24 @@ def get_queryset(self): 'scores__column', 'task', ) + elif self.action in ['delete_many', 're_run_many_submissions']: + try: + pks = list(self.request.data) + except TypeError as err: + raise ValidationError(f'Error {err}') + qs = qs.filter(pk__in=pks) + if not self.request.user.is_superuser and not self.request.user.is_staff: + if qs.filter( + Q(owner=self.request.user) | + Q(phase__competition__created_by=self.request.user) | + Q(phase__competition__collaborators__in=[self.request.user.pk]) + ) is not qs: + ValidationError("Request Contained Submissions you don't have authorization for") + if self.action in ['re_run_many_submissions']: + print(f'debug {qs}') + print(f'debug {qs.first().status}') + qs = qs.filter(status__in=[Submission.FINISHED, Submission.FAILED, Submission.CANCELLED]) + print(f'debug {qs}') return qs def create(self, request, *args, **kwargs): @@ -104,6 +122,14 @@ def destroy(self, request, *args, **kwargs): self.perform_destroy(submission) return Response(status=status.HTTP_204_NO_CONTENT) + @action(detail=False, methods=('DELETE',)) + def delete_many(self, request, *args, **kwargs): + qs = self.get_queryset() + if not qs: + return Response({'Submission search returned empty'}, status=status.HTTP_404_NOT_FOUND) + qs.delete() + return Response({}) + def get_renderer_context(self): """We override this to pass some context to the CSV renderer""" context = super().get_renderer_context() @@ -190,6 +216,13 @@ def re_run_submission(self, request, pk): new_sub = submission.re_run(**rerun_kwargs) return Response({'id': new_sub.id}) + @action(detail=False, methods=('POST',)) + def re_run_many_submissions(self, request): + qs = self.get_queryset() + for submission in qs: + submission.re_run() + return Response({}) + @action(detail=True, methods=('GET',)) def get_details(self, request, pk): submission = super().get_object() diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 60f2a0618..e8bbddaee 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -142,7 +142,7 @@ def _send_to_compute_worker(submission, is_scoring): run_args['detailed_results_url'] = make_url_sassy( path=submission.detailed_result.name, permission='w', - content_type='' + content_type='text/html' ) run_args['prediction_result'] = make_url_sassy( path=submission.prediction_result.name, diff --git a/src/settings/base.py b/src/settings/base.py index 843652c10..fa4f27aae 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -232,7 +232,7 @@ 'LOCATION': [os.environ.get("REDIS_URL", "redis://redis:6379")], 'OPTIONS': { "CONNECTION_POOL_KWARGS": { - "max_connections": 30, + "max_connections": os.environ.get("REDIS_MAX_CONNECTIONS", 20), "retry_on_timeout": True }, "COMPRESSOR": "django_redis.compressors.zlib.ZlibCompressor", diff --git a/src/static/img/codabench_black.png b/src/static/img/codabench_black.png new file mode 100644 index 000000000..9ec5f2648 Binary files /dev/null and b/src/static/img/codabench_black.png differ diff --git a/src/static/js/ours/client.js b/src/static/js/ours/client.js index 5a3b0626b..fd747afd1 100644 --- a/src/static/js/ours/client.js +++ b/src/static/js/ours/client.js @@ -85,6 +85,9 @@ CODALAB.api = { delete_submission: function (pk) { return CODALAB.api.request('DELETE', `${URLS.API}submissions/${pk}/`) }, + delete_many_submissions: function (pks) { + return CODALAB.api.request('DELETE', `${URLS.API}submissions/delete_many/`, pks) + }, toggle_submission_is_public: function (pk) { return CODALAB.api.request('GET', `${URLS.API}submissions/${pk}/toggle_public/`) }, @@ -94,6 +97,9 @@ CODALAB.api = { re_run_submission: function (id) { return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/re_run_submission/`) }, + re_run_many_submissions: function (data) { + return CODALAB.api.request('POST', `${URLS.API}submissions/re_run_many_submissions/`, data) + }, get_submission_csv_URL: function (filters) { filters.format = "csv" return `${URLS.API}submissions/?${$.param(filters)}` diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag index bcd6aab78..05dc2c3dd 100644 --- a/src/static/riot/competitions/detail/leaderboards.tag +++ b/src/static/riot/competitions/detail/leaderboards.tag @@ -8,7 +8,7 @@ - +
@@ -87,6 +87,7 @@ $('#search-leaderboard-button').click(function() { $(self.refs.leaderboardFilter).focus() }) + $('#leaderboardTable').tablesort() }) self.filter_columns = () => { @@ -140,6 +141,7 @@ } } self.filter_columns() + $('#leaderboardTable').tablesort() self.update() }) } diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index e555b9cac..3ec95f8e4 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -16,6 +16,16 @@ Download as CSV + +
@@ -41,6 +51,12 @@ + @@ -61,6 +77,12 @@ + @@ -169,6 +191,7 @@ self.hide_output = false self.leaderboards = {} self.loading = true + self.checked_submissions = [] self.on("mount", function () { $(self.refs.search).dropdown() @@ -217,6 +240,7 @@ } self.csv_link = CODALAB.api.get_submission_csv_URL(filters) self.update() + self.submission_checked() // Timeout here so loader doesn't flicker _.delay(() => { @@ -291,9 +315,24 @@ toastr.success('Submission queued') self.update_submissions() }) + .fail(function (response) { + if(response.responseJSON.detail){ + toastr.error(response.responseJSON.detail) + } else { + toastr.error(response.responseText) + } + }) event.stopPropagation() } + self.rerun_selected_submissions = function () { + CODALAB.api.re_run_many_submissions(self.checked_submissions) + .done(function (response) { + toastr.success('Submissions queued') + self.update_submissions() + }) + } + self.cancel_submission = function (submission) { CODALAB.api.cancel_submission(submission.id) .done(function (response) { @@ -318,6 +357,20 @@ event.stopPropagation() } + self.delete_selected_submissions = function () { + if (confirm(`Are you sure you want to delete the selected submissions?`)) { + console.log() + CODALAB.api.delete_many_submissions(self.checked_submissions) + .done(function (response) { + toastr.success('Submissions deleted') + self.update_submissions() + }) + .fail(function (response){ + toastr.error('Something went wrong') + }) + } + } + self.get_score_details = function (submission, column) { try { let score = _.filter(submission.scores, (score) => { @@ -346,6 +399,26 @@ } } + self.submission_checked = function () { + if (typeof(event) === "object" ){ + // We can't stop upon page load as there is no "event" (button click). + // We can when we we check the checkboxes as that is an "event". + event.stopPropagation() + } + let inputs = $(self.refs.submission_table).find('input') + let checked_boxes = inputs.not(':first').filter('input:checked') + let unchecked_boxes = inputs.not(':first').filter('input:not(:checked)') + inputs.first().prop('checked', unchecked_boxes.length === 0) + self.checked_submissions = checked_boxes.serializeArray().map((x) => {return x.name}) + } + + self.select_all_pressed = function () { + let check_boxes = $(self.refs.submission_table).find('input') + // Set checkboxes to be equal to Select_All checkbox + check_boxes.prop('checked', check_boxes.first().is(':checked')) + } + + self.submission_clicked = function (submission) { // stupid workaround to not modify the original submission object diff --git a/src/templates/pages/home.html b/src/templates/pages/home.html index 5bbaf1c4a..2164e4691 100644 --- a/src/templates/pages/home.html +++ b/src/templates/pages/home.html @@ -148,13 +148,13 @@

Codalab in Research

million Euro prize of the EU, organized by the See.4C consortium, the CIKM - + AnalytiCup 2017, which attracted 493 participants, MSCOCO (633 participants) and the ChaLearn - + AutoML challenge 2017 (687 participants).

Since 2016, Codalab offers the possibility of organizing machine learning challenges with @@ -309,7 +309,7 @@

Homework

simulators providing data on demand (with your own "ingestion program"); use the ChaLab wizard to create - competitions in minutes. + competitions in minutes. Thanks!

diff --git a/src/tests/functional/test_competitions.py b/src/tests/functional/test_competitions.py index 0d293beb0..99c2e34dc 100644 --- a/src/tests/functional/test_competitions.py +++ b/src/tests/functional/test_competitions.py @@ -95,6 +95,7 @@ def test_manual_competition_creation(self): self.find('i[class="add icon"]').click() self.find('input[selenium="title"]').send_keys('Title') self.execute_script('$("textarea[ref=\'content\']")[0].EASY_MDE.value("Testing123")') + sleep(1) self.find('div[selenium="save1"]').click() sleep(1)
+
+ + +
+ 		ID # File name Owner
+
+ + +
+ 		{ submission.id } { submission.filename } { submission.owner }