Merge develop into master

circle.yml → .circleci/config.yml

@@ -3,7 +3,7 @@ version: 2
 jobs:
   test:
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-2004:2022.07.1
     steps:
       - checkout
 

.env_sample

@@ -8,7 +8,7 @@ DB_PORT=5432
 
 DJANGO_SETTINGS_MODULE=settings.develop
 ALLOWED_HOSTS=localhost,example.com
-SUBMISSIONS_API_URL=http://example.com/api
+SUBMISSIONS_API_URL=http://localhost/api
 
 # Local domain definition
 DOMAIN_NAME=localhost:80
@@ -55,7 +55,7 @@ AWS_SECRET_ACCESS_KEY=testsecret
 AWS_STORAGE_BUCKET_NAME=public
 AWS_STORAGE_PRIVATE_BUCKET_NAME=private
 # NOTE! port 9000 here should match $MINIO_PORT
-AWS_S3_ENDPOINT_URL=http://docker.for.mac.localhost:9000/
+AWS_S3_ENDPOINT_URL=http://localhost:9000/
 AWS_QUERYSTRING_AUTH=False
 
 # # S3 storage example

Procfile

@@ -1,2 +1,2 @@
-web: cd src && gunicorn asgi:application -w 4 -k uvicorn.workers.UvicornWorker -b :$PORT --max-requests 1024 --max-requests-jitter 256
+web: cd src && gunicorn asgi:application -w 3 -k uvicorn.workers.UvicornWorker -b :$PORT --max-requests 1024 --max-requests-jitter 256
 worker: cd src && celery -A celery_config worker -B -Q site-worker -l info -n site-worker@%n --concurrency=3

README.md

@@ -1,6 +1,21 @@
-# Codabench
+![CodaBench logo](src/static/img/codabench_black.png) [![Circle CI](https://circleci.com/gh/codalab/codabench.svg?style=shield)](https://app.circleci.com/pipelines/github/codalab/codabench)
 
-## Installation
+## What is CodaBench?
+
+CodaBench is an open-source web-based platform that enables researchers, developers, and data scientists to collaborate, with the goal of advancing research fields where machine learning and advanced computation is used. CodaBench helps to solve many common problems in the arena of data-oriented research through its online community where people can share worksheets and participate in competitions and benchmarks. It can be seen as a version 2 of [CodaLab Competitions](https://github.com/codalab/codalab-competitions).
+
+To see CodaBench in action, visit [codabench.org](https://www.codabench.org/).
+
+
+## Documentation
+
+- [CodaBench Wiki](https://github.com/codalab/codabench/wiki)
+
+
+## Quick installation (for Linux)
+
+_To participate, or even organize your own benchmarks or competitions, **you don't need to install anything**, you just need to sign in an instance of the platform (e.g. [this one](https://www.codabench.org/)). 
+If you wish to configure your own instance of CodaBench platform, here are the instructions:_
 
 
 ```
@@ -15,6 +30,34 @@ You can now login as username "admin" with password "admin" at http://localhost:
 
 If you ever need to reset the database, use the script `./reset_db.sh`
 
+
+## License
+
+Copyright (c) 2020-2022, Université Paris-Saclay.
+This software is released under the Apache License 2.0 (the "License"); you may not use the software except in compliance with the License.
+
+The text of the Apache License 2.0 can be found online at:
+http://www.opensource.org/licenses/apache2.0.php
+
+
+## Cite CodaBench in your research
+
+```
+@article{codabench,
+    title = {Codabench: Flexible, easy-to-use, and reproducible meta-benchmark platform},
+    author = {Zhen Xu and Sergio Escalera and Adrien Pavão and Magali Richard and 
+              Wei-Wei Tu and Quanming Yao and Huan Zhao and Isabelle Guyon},
+    journal = {Patterns},
+    volume = {3},
+    number = {7},
+    pages = {100543},
+    year = {2022},
+    issn = {2666-3899},
+    doi = {https://doi.org/10.1016/j.patter.2022.100543},
+    url = {https://www.sciencedirect.com/science/article/pii/S2666389922001465}
+}
+```
+
 ## Running tests
 
 ```
@@ -177,3 +220,4 @@ You can execute commands against a role:
 ```
 
 See available commands with `fab -l`
+

bin/pg_dump.py

@@ -2,7 +2,7 @@
 """
 Usage, in `crontab -e`:
 
-    @daily /home/ubuntu/competitions-v2/bin/pg_dump.py
+    @daily /home/ubuntu/codabench/bin/pg_dump.py
 
 """
 import time
@@ -17,13 +17,15 @@
 call([
     'docker',
     'exec',
-    'competitions-v2_db_1',
+    'codabench-db-1',
     'bash',
     '-c',
-    f'PGPASSWORD=$DB_PASSWORD pg_dump -Fc -U $DB_USER $DB_NAME > /app/backups/{dump_name}'
+    f'PGPASSWORD=$DB_PASSWORD pg_dump -Fc -U $DB_USERNAME $DB_NAME > /app/backups/{dump_name}'
 ])
 
 # Push/destroy dump
 call([
-    'docker', 'exec', 'competitions-v2_django_1', 'python', 'manage.py', 'upload_backup', f'{dump_name}'
+    'docker', 'exec', 'codabench-django-1', 'python', 'manage.py', 'upload_backup', f'{dump_name}'
 ])
+
+

docker/compute_worker/compute_worker.py

@@ -261,7 +261,7 @@ def get_detailed_results_file_path(self):
 
     async def send_detailed_results(self, file_path):
         logger.info(f"Updating detailed results {file_path} - {self.detailed_results_url}")
-        self._put_file(self.detailed_results_url, file=file_path, content_type='')
+        self._put_file(self.detailed_results_url, file=file_path, content_type='text/html')
         async with websockets.connect(self.websocket_url) as websocket:
             await websocket.send(json.dumps({
                 "kind": 'detailed_result_update',
@@ -734,7 +734,7 @@ def push_scores(self):
         elif os.path.exists(os.path.join(self.output_dir, "scores.txt")):
             scores_file = os.path.join(self.output_dir, "scores.txt")
             with open(scores_file) as f:
-                scores = yaml.load(f)
+                scores = yaml.load(f, yaml.Loader)
         else:
             raise SubmissionException("Could not find scores file, did the scoring program output it?")
 

src/apps/api/serializers/leaderboards.py

@@ -128,7 +128,7 @@ def get_columns(self, instance):
         if len(columns) == 0:
             raise serializers.ValidationError("No columns exist on the leaderboard")
         else:
-            return ColumnSerializer(columns, many=len(columns) > 1).data
+            return ColumnSerializer(columns, many=len(columns) >= 1).data
 
     class Meta:
         model = Phase

src/apps/api/views/submissions.py

@@ -82,6 +82,24 @@ def get_queryset(self):
                 'scores__column',
                 'task',
             )
+        elif self.action in ['delete_many', 're_run_many_submissions']:
+            try:
+                pks = list(self.request.data)
+            except TypeError as err:
+                raise ValidationError(f'Error {err}')
+            qs = qs.filter(pk__in=pks)
+            if not self.request.user.is_superuser and not self.request.user.is_staff:
+                if qs.filter(
+                    Q(owner=self.request.user) |
+                    Q(phase__competition__created_by=self.request.user) |
+                    Q(phase__competition__collaborators__in=[self.request.user.pk])
+                ) is not qs:
+                    ValidationError("Request Contained Submissions you don't have authorization for")
+            if self.action in ['re_run_many_submissions']:
+                print(f'debug {qs}')
+                print(f'debug {qs.first().status}')
+                qs = qs.filter(status__in=[Submission.FINISHED, Submission.FAILED, Submission.CANCELLED])
+                print(f'debug {qs}')
         return qs
 
     def create(self, request, *args, **kwargs):
@@ -104,6 +122,14 @@ def destroy(self, request, *args, **kwargs):
         self.perform_destroy(submission)
         return Response(status=status.HTTP_204_NO_CONTENT)
 
+    @action(detail=False, methods=('DELETE',))
+    def delete_many(self, request, *args, **kwargs):
+        qs = self.get_queryset()
+        if not qs:
+            return Response({'Submission search returned empty'}, status=status.HTTP_404_NOT_FOUND)
+        qs.delete()
+        return Response({})
+
     def get_renderer_context(self):
         """We override this to pass some context to the CSV renderer"""
         context = super().get_renderer_context()
@@ -190,6 +216,13 @@ def re_run_submission(self, request, pk):
         new_sub = submission.re_run(**rerun_kwargs)
         return Response({'id': new_sub.id})
 
+    @action(detail=False, methods=('POST',))
+    def re_run_many_submissions(self, request):
+        qs = self.get_queryset()
+        for submission in qs:
+            submission.re_run()
+        return Response({})
+
     @action(detail=True, methods=('GET',))
     def get_details(self, request, pk):
         submission = super().get_object()

src/apps/competitions/tasks.py

@@ -142,7 +142,7 @@ def _send_to_compute_worker(submission, is_scoring):
             run_args['detailed_results_url'] = make_url_sassy(
                 path=submission.detailed_result.name,
                 permission='w',
-                content_type=''
+                content_type='text/html'
             )
         run_args['prediction_result'] = make_url_sassy(
             path=submission.prediction_result.name,

src/settings/base.py

@@ -232,7 +232,7 @@
         'LOCATION': [os.environ.get("REDIS_URL", "redis://redis:6379")],
         'OPTIONS': {
             "CONNECTION_POOL_KWARGS": {
-                "max_connections": 30,
+                "max_connections": os.environ.get("REDIS_MAX_CONNECTIONS", 20),
                 "retry_on_timeout": True
             },
             "COMPRESSOR": "django_redis.compressors.zlib.ZlibCompressor",

src/static/js/ours/client.js

@@ -85,6 +85,9 @@ CODALAB.api = {
     delete_submission: function (pk) {
         return CODALAB.api.request('DELETE', `${URLS.API}submissions/${pk}/`)
     },
+    delete_many_submissions: function (pks) {
+        return CODALAB.api.request('DELETE', `${URLS.API}submissions/delete_many/`, pks)
+    },
     toggle_submission_is_public: function (pk) {
         return CODALAB.api.request('GET', `${URLS.API}submissions/${pk}/toggle_public/`)
     },
@@ -94,6 +97,9 @@ CODALAB.api = {
     re_run_submission: function (id) {
         return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/re_run_submission/`)
     },
+    re_run_many_submissions: function (data) {
+        return CODALAB.api.request('POST', `${URLS.API}submissions/re_run_many_submissions/`, data)
+    },
     get_submission_csv_URL: function (filters) {
         filters.format = "csv"
         return `${URLS.API}submissions/?${$.param(filters)}`

src/static/riot/competitions/detail/leaderboards.tag

@@ -8,7 +8,7 @@
     <a data-tooltip="Start typing to filter columns under 'Meta-data' or Tasks." data-position="right center">
         <i class="grey question circle icon"></i>
     </a>
-    <table id="leadboardTable" class="ui celled selectable table">
+    <table id="leaderboardTable" class="ui celled selectable sortable table">
         <thead>
         <tr>
             <th colspan="100%" class="center aligned">
@@ -87,6 +87,7 @@
             $('#search-leaderboard-button').click(function() {
                 $(self.refs.leaderboardFilter).focus()
             })
+            $('#leaderboardTable').tablesort()
         })
 
         self.filter_columns = () => {
@@ -140,6 +141,7 @@
                         }
                     }
                     self.filter_columns()
+                    $('#leaderboardTable').tablesort()
                     self.update()
                 })
         }

src/static/riot/competitions/detail/submission_manager.tag

@@ -16,6 +16,16 @@
         <a class="ui button" href="{csv_link}">
             <i class="icon download"></i>Download as CSV
         </a>
+        <button type="button" class="ui button right floated" disabled="{checked_submissions.length === 0}"
+                onclick="{delete_selected_submissions.bind(this)}">
+            <i class="icon trash alternate"></i>
+            Delete Submissions
+        </button>
+        <button type="button" class="ui button right floated" disabled="{checked_submissions.length === 0}"
+                onclick="{rerun_selected_submissions.bind(this)}">
+            <i class="icon redo"></i>
+            Rerun Submissions
+        </button>
     </div>
     <div class="ui icon input">
         <input type="text" placeholder="Search..." ref="search" onkeyup="{ filter }">
@@ -41,6 +51,12 @@
     <table class="ui celled selectable sortable table" ref="submission_table">
         <thead>
         <tr>
+            <th if="{opts.admin}">
+                <div class="ui checkbox" onclick="{select_all_pressed.bind(this)}">
+                    <input type="checkbox" name="select_all">
+                    <label>All</label>
+                </div>
+            </th>
             <th class="sorted descending collapsing">ID #</th>
             <th>File name</th>
             <th if="{ opts.admin }">Owner</th>
@@ -61,6 +77,12 @@
         </tr>
         <tr show="{!loading}" each="{ submission, index in filter_children(submissions) }"
             onclick="{ submission_clicked.bind(this, submission) }" class="submission_row">
+            <td if="{opts.admin}">
+                <div class="ui checkbox" onclick="{submission_checked.bind(this)}">
+                    <input type="checkbox" name="{submission.id}">
+                    <label></label>
+                </div>
+            </td>
             <td>{ submission.id }</td>
             <td>{ submission.filename }</td>
             <td if="{ opts.admin }">{ submission.owner }</td>
@@ -169,6 +191,7 @@
         self.hide_output = false
         self.leaderboards = {}
         self.loading = true
+        self.checked_submissions = []
 
         self.on("mount", function () {
             $(self.refs.search).dropdown()
@@ -217,6 +240,7 @@
                     }
                     self.csv_link = CODALAB.api.get_submission_csv_URL(filters)
                     self.update()
+                    self.submission_checked()
 
                     // Timeout here so loader doesn't flicker
                     _.delay(() => {
@@ -291,9 +315,24 @@
                     toastr.success('Submission queued')
                     self.update_submissions()
                 })
+                .fail(function (response) {
+                    if(response.responseJSON.detail){
+                        toastr.error(response.responseJSON.detail)
+                    } else {
+                        toastr.error(response.responseText)
+                    }
+                })
             event.stopPropagation()
         }
 
+        self.rerun_selected_submissions = function () {
+            CODALAB.api.re_run_many_submissions(self.checked_submissions)
+                .done(function (response) {
+                    toastr.success('Submissions queued')
+                    self.update_submissions()
+                })
+        }
+
         self.cancel_submission = function (submission) {
             CODALAB.api.cancel_submission(submission.id)
                 .done(function (response) {
@@ -318,6 +357,20 @@
             event.stopPropagation()
         }
 
+        self.delete_selected_submissions = function () {
+            if (confirm(`Are you sure you want to delete the selected submissions?`)) {
+                console.log()
+                CODALAB.api.delete_many_submissions(self.checked_submissions)
+                    .done(function (response) {
+                        toastr.success('Submissions deleted')
+                        self.update_submissions()
+                    })
+                    .fail(function (response){
+                        toastr.error('Something went wrong')
+                    })
+            }
+            }
+
         self.get_score_details = function (submission, column) {
             try {
                 let score = _.filter(submission.scores, (score) => {
@@ -346,6 +399,26 @@
             }
         }
 
+        self.submission_checked = function () {
+            if (typeof(event) === "object" ){
+                // We can't stop upon page load as there is no "event" (button click).
+                // We can when we we check the checkboxes as that is an "event". 
+                event.stopPropagation() 
+            }
+            let inputs = $(self.refs.submission_table).find('input')
+            let checked_boxes = inputs.not(':first').filter('input:checked')
+            let unchecked_boxes = inputs.not(':first').filter('input:not(:checked)')
+            inputs.first().prop('checked', unchecked_boxes.length === 0)
+            self.checked_submissions = checked_boxes.serializeArray().map((x) => {return x.name})
+        }
+
+        self.select_all_pressed = function () {
+            let check_boxes = $(self.refs.submission_table).find('input')
+            // Set checkboxes to be equal to Select_All checkbox
+            check_boxes.prop('checked', check_boxes.first().is(':checked'))
+        }
+
+
 
         self.submission_clicked = function (submission) {
             // stupid workaround to not modify the original submission object