From 72269ddd7137a3aa65530c0bd94e44a0e809da87 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 16 Aug 2023 16:28:07 +0500 Subject: [PATCH 001/136] do not allow users to rerun all submissions if submissions are more than 200. Only super admin can rerun in this case. --- src/apps/api/views/competitions.py | 15 +++++++++++++++ .../competitions/detail/submission_manager.tag | 3 +++ 2 files changed, 18 insertions(+) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 8fcb69a4b..b082a8997 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -583,13 +583,28 @@ def manually_migrate(self, request, pk): @action(detail=True, url_name='rerun_submissions') def rerun_submissions(self, request, pk): + + # Limit for rerunning submissions + RERUN_SUBMISSION_LIMIT = 200 + phase = self.get_object() comp = phase.competition + + # error when user is not super user or admin of the competition if request.user not in comp.all_organizers and not request.user.is_superuser: raise PermissionDenied('You do not have permission to re-run submissions') + + # Get submissions submissions = phase.submissions.all() + + # error when user is not super user and submissions crosses the limit + if not request.user.is_superuser and len(submissions) >= RERUN_SUBMISSION_LIMIT: + raise PermissionDenied('You have too many submissions, Contact us on `info@codalab.org` to request a rerun.') + + # rerun all submissions for submission in submissions: submission.re_run() + rerun_count = len(submissions) return Response({"count": rerun_count}) diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 057cd089b..7b9aa43d7 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -284,6 +284,9 @@ toastr.success(`Rerunning ${response.count} submissions`) self.update_submissions() }) + .fail(function (response) { + toastr.error(response.responseJSON.detail) + }) } } self.filter = function () { From 5b638eacb42adf566877eebc8147b0580a58dde2 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 16 Aug 2023 19:02:21 +0500 Subject: [PATCH 002/136] additional conditions added for queue owner and organizer --- src/apps/api/views/competitions.py | 63 ++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index b082a8997..b2449c07b 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -585,28 +585,69 @@ def manually_migrate(self, request, pk): def rerun_submissions(self, request, pk): # Limit for rerunning submissions - RERUN_SUBMISSION_LIMIT = 200 + RERUN_SUBMISSION_LIMIT = 30 phase = self.get_object() comp = phase.competition - # error when user is not super user or admin of the competition - if request.user not in comp.all_organizers and not request.user.is_superuser: - raise PermissionDenied('You do not have permission to re-run submissions') - # Get submissions submissions = phase.submissions.all() + can_re_run_submissions = False + error_message = "" + + # Super admin can rerun without any restrictions + if request.user.is_superuser: + can_re_run_submissions = True + + # competition admin can run only if + elif request.user in comp.all_organizers: + + # submissions are in limit + if len(submissions) <= RERUN_SUBMISSION_LIMIT: + can_re_run_submissions = True + + # submissions are not in limit + else: + # Codabemch public queue + if comp.queue is None: + can_re_run_submissions = False + error_message = f"You cannot rerun more than {RERUN_SUBMISSION_LIMIT} submissions on Codabench public queue! Contact us on `info@codalab.org` to request a rerun." + + # Other queue where user is not owner and not organizer + elif request.user != comp.queue.owner and request.user not in comp.queue.organizers.all(): + can_re_run_submissions = False + error_message = f"You cannot rerun more than {RERUN_SUBMISSION_LIMIT} submissions on a queue which is not yours! Contact us on `info@codalab.org` to request a rerun." + + # User can rerun submissions where he is owner or organizer + else: + can_re_run_submissions = True + + else: + can_re_run_submissions = False + error_message = 'You do not have permission to re-run submissions' + + # error when user is not super user or admin of the competition + if can_re_run_submissions: + # rerun all submissions + # for submission in submissions: + # submission.re_run() + rerun_count = len(submissions) + return Response({"count": rerun_count}) + else: + raise PermissionDenied(error_message) + + + + + + + # error when user is not super user and submissions crosses the limit if not request.user.is_superuser and len(submissions) >= RERUN_SUBMISSION_LIMIT: raise PermissionDenied('You have too many submissions, Contact us on `info@codalab.org` to request a rerun.') - # rerun all submissions - for submission in submissions: - submission.re_run() - - rerun_count = len(submissions) - return Response({"count": rerun_count}) + @swagger_auto_schema(responses={200: PhaseResultsSerializer}) @action(detail=True, methods=['GET']) From bd3beddc871925dfc1073e170534f94ccbe66990 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 16 Aug 2023 19:07:28 +0500 Subject: [PATCH 003/136] empty lines removed --- src/apps/api/views/competitions.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index b2449c07b..6902a8296 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -637,18 +637,6 @@ def rerun_submissions(self, request, pk): else: raise PermissionDenied(error_message) - - - - - - - # error when user is not super user and submissions crosses the limit - if not request.user.is_superuser and len(submissions) >= RERUN_SUBMISSION_LIMIT: - raise PermissionDenied('You have too many submissions, Contact us on `info@codalab.org` to request a rerun.') - - - @swagger_auto_schema(responses={200: PhaseResultsSerializer}) @action(detail=True, methods=['GET']) def get_leaderboard(self, request, pk): From 66d66b4cae07cc601a2db20e2a8add1ec0ea5e5b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 16 Aug 2023 19:22:39 +0500 Subject: [PATCH 004/136] rerun code uncommented --- src/apps/api/views/competitions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 6902a8296..5602204a0 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -630,8 +630,8 @@ def rerun_submissions(self, request, pk): # error when user is not super user or admin of the competition if can_re_run_submissions: # rerun all submissions - # for submission in submissions: - # submission.re_run() + for submission in submissions: + submission.re_run() rerun_count = len(submissions) return Response({"count": rerun_count}) else: From f92e15f209619ea65da8beef926c173f9f59cf35 Mon Sep 17 00:00:00 2001 From: bbearce Date: Sun, 10 Sep 2023 20:54:39 -0400 Subject: [PATCH 005/136] blessings for shell_plus --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5a14473b8..291f4d483 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,6 +26,7 @@ bleach==3.1.4 # Heroku staging debug tools django-debug-toolbar==3.2 django-querycount==0.7.0 +blessings==1.7 # User impersonation django-su==0.9.0 From 318db685e69ab5860ca7aec22f37b4a44cd0ac34 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 11 Sep 2023 18:57:02 +0500 Subject: [PATCH 006/136] http instead of https --- src/utils/context_processors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/context_processors.py b/src/utils/context_processors.py index 74213b888..4b6d77fab 100644 --- a/src/utils/context_processors.py +++ b/src/utils/context_processors.py @@ -21,6 +21,6 @@ def common_settings(request): return { 'STORAGE_TYPE': settings.STORAGE_TYPE, 'USER_JSON_DATA': json.dumps(user_json_data), - 'RABBITMQ_MANAGEMENT_URL': f"https://{settings.DOMAIN_NAME}:{settings.RABBITMQ_MANAGEMENT_PORT}", - 'FLOWER_URL': f"https://{settings.DOMAIN_NAME}:{settings.FLOWER_PUBLIC_PORT}", + 'RABBITMQ_MANAGEMENT_URL': f"http://{settings.DOMAIN_NAME}:{settings.RABBITMQ_MANAGEMENT_PORT}", + 'FLOWER_URL': f"http://{settings.DOMAIN_NAME}:{settings.FLOWER_PUBLIC_PORT}", } From 438de4d1911e251d132f0612a3d2d4d43859c0d6 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 14 Sep 2023 16:08:28 +0500 Subject: [PATCH 007/136] task update and delete option visible to owner only --- src/apps/api/serializers/tasks.py | 1 + src/static/riot/tasks/management.tag | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/apps/api/serializers/tasks.py b/src/apps/api/serializers/tasks.py index b98ce36ea..1526a1980 100644 --- a/src/apps/api/serializers/tasks.py +++ b/src/apps/api/serializers/tasks.py @@ -138,6 +138,7 @@ class Meta: fields = ( 'id', 'created_when', + 'created_by', 'key', 'name', 'solutions', diff --git a/src/static/riot/tasks/management.tag b/src/static/riot/tasks/management.tag index 5a613b66a..06d24bb2f 100644 --- a/src/static/riot/tasks/management.tag +++ b/src/static/riot/tasks/management.tag @@ -45,15 +45,17 @@ - - +
+ + +
-
+
From 2fce7d6eff5f73f242a5ae5c3fd56b86b300ce39 Mon Sep 17 00:00:00 2001 From: didayolo Date: Thu, 14 Sep 2023 13:50:25 +0200 Subject: [PATCH 008/136] Add help in Files table --- src/static/riot/competitions/detail/_tabs.tag | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/_tabs.tag b/src/static/riot/competitions/detail/_tabs.tag index dccee3d94..74ba34972 100644 --- a/src/static/riot/competitions/detail/_tabs.tag +++ b/src/static/riot/competitions/detail/_tabs.tag @@ -60,7 +60,12 @@ Phase Task Type - Available + Available + + + Size From b568f68cf5bce8b2e24a885ca657844347baf34d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 18 Sep 2023 01:38:42 +0500 Subject: [PATCH 009/136] compute worker crash handled: docker image pull and execution time limit --- compute_worker/compute_worker.py | 104 +++++++++++++++++++++++++++--- src/apps/api/views/submissions.py | 44 ++++++++++++- 2 files changed, 137 insertions(+), 11 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 875051090..5df5995eb 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -26,9 +26,12 @@ from kombu import Queue, Exchange from urllib3 import Retry - logger = logging.getLogger() + +# ----------------------------------------------- +# Celery + Rabbit MQ +# ----------------------------------------------- # Init celery + rabbit queue definitions app = Celery() app.config_from_object('celery_config') # grabs celery_config.py @@ -38,6 +41,9 @@ ] +# ----------------------------------------------- +# Directories +# ----------------------------------------------- # Setup base directories used by all submissions # note: we need to pass this directory to docker-compose so it knows where to store things! HOST_DIRECTORY = os.environ.get("HOST_DIRECTORY", "/tmp/codabench/") @@ -45,6 +51,10 @@ CACHE_DIR = os.path.join(BASE_DIR, "cache") MAX_CACHE_DIR_SIZE_GB = float(os.environ.get('MAX_CACHE_DIR_SIZE_GB', 10)) + +# ----------------------------------------------- +# Submission status +# ----------------------------------------------- # Status options for submissions STATUS_NONE = "None" STATUS_SUBMITTING = "Submitting" @@ -65,6 +75,10 @@ STATUS_FAILED, ) + +# ----------------------------------------------- +# Container Engine +# ----------------------------------------------- # Setup the container engine that we are using if os.environ.get("CONTAINER_ENGINE_EXECUTABLE"): CONTAINER_ENGINE_EXECUTABLE = os.environ.get("CONTAINER_ENGINE_EXECUTABLE") @@ -75,9 +89,18 @@ CONTAINER_ENGINE_EXECUTABLE = "docker" +# ----------------------------------------------- +# Exceptions +# ----------------------------------------------- class SubmissionException(Exception): pass +class DockerImagePullException(Exception): + pass + +class ExecutionTimeLimitExceeded(Exception): + pass + # ----------------------------------------------------------------------------- # The main compute worker entrypoint, this is how a job is ran at the highest @@ -94,6 +117,8 @@ def run_wrapper(run_args): if run.is_scoring: run.push_scores() run.push_output() + except DockerImagePullException as e: + run._update_status(STATUS_FAILED, str(e)) except SubmissionException as e: run._update_status(STATUS_FAILED, str(e)) except SoftTimeLimitExceeded: @@ -160,14 +185,14 @@ def is_valid_zip(zip_path): return False -class ExecutionTimeLimitExceeded(Exception): - pass - - def alarm_handler(signum, frame): raise ExecutionTimeLimitExceeded +# ----------------------------------------------- +# Class Run +# Respnosible for running a submission inside a docker container +# ----------------------------------------------- class Run: """A "Run" in Codalab is composed of some program, some data to work with, and some signed URLs to upload results to. There is also a secret key to do special commands for just this submission. @@ -340,8 +365,55 @@ def _get_container_image(self, image_name): container_engine_pull = check_output(cmd) logger.info("Pull complete for image: {0} with output of {1}".format(image_name, container_engine_pull)) except CalledProcessError: - logger.info("Pull for image: {} returned a non-zero exit code!") - raise SubmissionException(f"Pull for {image_name} failed!") + error_message = f"Pull for image: {image_name} returned a non-zero exit code! Check if the docker image exists on docker hub." + logger.info(error_message) + # Prepare data to be sent to submissions api + docker_pull_fail_data = { + "type": "Docker_Image_Pull_Fail", + "error_message": error_message, + } + # Send data to be written to ingestion logs + self._update_submission(docker_pull_fail_data) + # Send error through web socket to the frontend + asyncio.run(self._send_data_through_socket(error_message)) + raise DockerImagePullException(f"Pull for {image_name} failed!") + + async def _send_data_through_socket(self, error_message): + """ + This function gets an error messages and sends it through a web socket. This function is used for sending + - Docker image pull failure logs + - Execution time limit exceeded logs + """ + logger.info(f"Connecting to {self.websocket_url} to send docker image pull error") + + # connect to web socket + websocket = await websockets.connect(self.websocket_url) + + # define websocket errors + websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError) + + try: + # send message + await websocket.send(json.dumps({ + "kind": "stderr", + "message": error_message + })) + + except websocket_errors: + # handle websocket errors + logger.info(f"Error sending failed through websocket") + try: + await websocket.close() + except Exception as e: + logger.error(e) + else: + # no error in websocket message sending + logger.info(f"Error sent successfully through websocket") + + logger.info(f"Disconnecting from websocket {self.websocket_url}") + + # close websocket + await websocket.close() def _get_bundle(self, url, destination, cache=True): """Downloads zip from url and unzips into destination. If cache=True then url is hashed and checked @@ -384,7 +456,7 @@ def _get_bundle(self, url, destination, cache=True): raise # Re-raise the last caught BadZipFile exception else: logger.info("Failed. Retrying in 60 seconds...") - time.sleep(60) # Wait 60 seconds before retrying + time.sleep(60) # Wait 60 seconds before retrying # Return the zip file path for other uses, e.g. for creating a MD5 hash to identify it return bundle_file @@ -627,7 +699,7 @@ def _put_file(self, url, file=None, raw_data=None, content_type='application/zip """ if file and raw_data: raise Exception("Cannot put both a file and raw_data") - + headers = { # For Azure only, other systems ignore these headers 'x-ms-blob-type': 'BlockBlob', @@ -731,7 +803,19 @@ def start(self): try: loop.run_until_complete(gathered_tasks) except ExecutionTimeLimitExceeded: - raise SubmissionException(f"Execution Time Limit exceeded. Limit was {self.execution_time_limit} seconds") + error_message = f"Execution Time Limit exceeded. Limit was {self.execution_time_limit} seconds" + logger.info(error_message) + # Prepare data to be sent to submissions api + execution_time_limit_exceeded_data = { + "type": "Execution_Time_Limit_Exceeded", + "error_message": error_message, + "is_scoring": self.is_scoring + } + # Send data to be written to ingestion/scoring std_err + self._update_submission(execution_time_limit_exceeded_data) + # Send error through web socket to the frontend + asyncio.run(self._send_data_through_socket(error_message)) + raise SubmissionException(error_message) finally: self.watch = False for kind, logs in self.logs.items(): diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index d4d82fba2..6ce9da84b 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -1,5 +1,6 @@ import json import uuid +import logging from django.db.models import Q from django_filters.rest_framework import DjangoFilterBackend @@ -14,14 +15,17 @@ from rest_framework.settings import api_settings from rest_framework.viewsets import ModelViewSet from rest_framework_csv import renderers +from django.core.files.base import ContentFile from profiles.models import Organization, Membership from tasks.models import Task from api.serializers.submissions import SubmissionCreationSerializer, SubmissionSerializer, SubmissionFilesSerializer -from competitions.models import Submission, Phase, CompetitionParticipant +from competitions.models import Submission, SubmissionDetails, Phase, CompetitionParticipant from leaderboards.strategies import put_on_leaderboard_by_submission_rule from leaderboards.models import SubmissionScore, Column, Leaderboard +logger = logging.getLogger() + class SubmissionViewSet(ModelViewSet): queryset = Submission.objects.all().order_by('-pk') @@ -50,6 +54,44 @@ def check_object_permissions(self, request, obj): hostname = request.data['status_details'].replace('scoring_hostname-', '') obj.scoring_worker_hostname = hostname obj.save() + + # check if type is in request data. type can have the following values + # - Docker_Image_Pull_Fail + # - Execution_Time_Limit_Exceeded + if "type" in self.request.data.keys(): + + if request.data["type"] in ["Docker_Image_Pull_Fail", "Execution_Time_Limit_Exceeded"]: + + # Get the error message + error_message = request.data['error_message'] + + # Set file name to ingestion std error as default + error_file_name = "prediction_ingestion_stderr" + + # Change error file name when error comes from execution time limit + # and error occured during scoring + if request.data["type"] == "Execution_Time_Limit_Exceeded" and request.data['is_scoring'] == "True": + error_file_name = "scoring_stderr" + + try: + # Get submission detail for this submission + submission_detail = SubmissionDetails.objects.get( + name=error_file_name, + submission=obj, + ) + + # Read the existing content from the file + existing_content = submission_detail.data_file.read().decode("utf-8") + + # Append the new error message to the existing content + modified_content = existing_content + "\n" + error_message + + # write error message to the file + submission_detail.data_file.save(submission_detail.data_file.name, ContentFile(modified_content.encode("utf-8"))) + + except SubmissionDetails.DoesNotExist: + logger.warning("SubmissionDetails object not found.") + not_bot_user = self.request.user.is_authenticated and not self.request.user.is_bot if self.action in ['update_fact_sheet', 're_run_submission']: From 0e1511e975e80d145ea77b779d8f1a7e9eb4279b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 18 Sep 2023 21:25:50 +0500 Subject: [PATCH 010/136] env variable added --- .env_sample | 8 ++++++++ src/apps/api/views/competitions.py | 10 ++++------ src/settings/base.py | 7 +++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.env_sample b/.env_sample index ce6916ef1..84a0888dc 100644 --- a/.env_sample +++ b/.env_sample @@ -59,6 +59,14 @@ AWS_STORAGE_PRIVATE_BUCKET_NAME=private AWS_S3_ENDPOINT_URL=http://minio:9000/ AWS_QUERYSTRING_AUTH=False +# ----------------------------------------------------------------------------- +# Limit for re-running submission +# This is used to limit users to rerun submissions +# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT +# ----------------------------------------------------------------------------- +RERUN_SUBMISSION_LIMIT=30 + + # # S3 storage example # STORAGE_TYPE=s3 # AWS_ACCESS_KEY_ID=12312312312312312331223 diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 5602204a0..a68052f5b 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -38,6 +38,7 @@ from api.permissions import IsOrganizerOrCollaborator from datetime import datetime from django.db import transaction +from django.conf import settings class CompetitionViewSet(ModelViewSet): @@ -584,9 +585,6 @@ def manually_migrate(self, request, pk): @action(detail=True, url_name='rerun_submissions') def rerun_submissions(self, request, pk): - # Limit for rerunning submissions - RERUN_SUBMISSION_LIMIT = 30 - phase = self.get_object() comp = phase.competition @@ -604,7 +602,7 @@ def rerun_submissions(self, request, pk): elif request.user in comp.all_organizers: # submissions are in limit - if len(submissions) <= RERUN_SUBMISSION_LIMIT: + if len(submissions) <= settings.RERUN_SUBMISSION_LIMIT: can_re_run_submissions = True # submissions are not in limit @@ -612,12 +610,12 @@ def rerun_submissions(self, request, pk): # Codabemch public queue if comp.queue is None: can_re_run_submissions = False - error_message = f"You cannot rerun more than {RERUN_SUBMISSION_LIMIT} submissions on Codabench public queue! Contact us on `info@codalab.org` to request a rerun." + error_message = f"You cannot rerun more than {settings.RERUN_SUBMISSION_LIMIT} submissions on Codabench public queue! Contact us on `info@codalab.org` to request a rerun." # Other queue where user is not owner and not organizer elif request.user != comp.queue.owner and request.user not in comp.queue.organizers.all(): can_re_run_submissions = False - error_message = f"You cannot rerun more than {RERUN_SUBMISSION_LIMIT} submissions on a queue which is not yours! Contact us on `info@codalab.org` to request a rerun." + error_message = f"You cannot rerun more than {settings.RERUN_SUBMISSION_LIMIT} submissions on a queue which is not yours! Contact us on `info@codalab.org` to request a rerun." # User can rerun submissions where he is owner or organizer else: diff --git a/src/settings/base.py b/src/settings/base.py index 1366a0cc8..b0751758e 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -447,3 +447,10 @@ # Django-Su (User impersonation) SU_LOGIN_CALLBACK = 'profiles.admin.su_login_callback' AJAX_LOOKUP_CHANNELS = {'django_su': dict(model='profiles.User', search_field='username')} + +# ============================================================================= +# Limit for re-running submission +# This is used to limit users to rerun submissions +# on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT +# ============================================================================= +RERUN_SUBMISSION_LIMIT = os.environ.get('RERUN_SUBMISSION_LIMIT', 30) From 99c14b9e61a5c2cf9cdc84f344761e0626f2ae57 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 19 Sep 2023 00:00:54 +0500 Subject: [PATCH 011/136] Phase dates ui bug removed --- .../riot/competitions/editor/_phases.tag | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/static/riot/competitions/editor/_phases.tag b/src/static/riot/competitions/editor/_phases.tag index 748c2609f..002bdf7a5 100644 --- a/src/static/riot/competitions/editor/_phases.tag +++ b/src/static/riot/competitions/editor/_phases.tag @@ -389,6 +389,25 @@ $(self.refs.modal).modal('hide') } + self.formatDateToYYYYMMDD = function(input) { + // This function formats date in the format YYYY-MM-DD + + // convert input to date + var dateObject = new Date(input) + + // check if date has a time + if (!isNaN(dateObject.getTime())) { + // Extract year + var year = dateObject.getFullYear() + // Extract Month + var month = (dateObject.getMonth() + 1).toString().padStart(2, '0') + // Extract day + var day = dateObject.getDate().toString().padStart(2, '0') + return `${year}-${month}-${day}` + } + return input + } + self.form_updated = function () { // This checks phases overall to make sure they are ready to go var is_valid = true @@ -406,9 +425,8 @@ }) _.forEach(_.range(self.phases.length), i => { if (i !== 0) { - let end = Date.parse(self.phases[i - 1].end) - let start = Date.parse(self.phases[i].start) - + let end = Date.parse(self.formatDateToYYYYMMDD(self.phases[i - 1].end)) + let start = Date.parse(self.formatDateToYYYYMMDD(self.phases[i].start)) if (end > start || !end) { let message = `Phase "${_.get(self.phases[i], 'name', i + 1)}" must start after phase "${_.get(self.phases[i - 1], 'name', i)}" ends` if (!self.warnings.includes(message)) { From 2b43568655e6e51743a8eb200a68584718cb2c88 Mon Sep 17 00:00:00 2001 From: Benjamin Bearce Date: Mon, 18 Sep 2023 19:52:36 +0000 Subject: [PATCH 012/136] ghost files fix --- src/static/riot/competitions/detail/_tabs.tag | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/static/riot/competitions/detail/_tabs.tag b/src/static/riot/competitions/detail/_tabs.tag index dccee3d94..d029e2a61 100644 --- a/src/static/riot/competitions/detail/_tabs.tag +++ b/src/static/riot/competitions/detail/_tabs.tag @@ -244,27 +244,28 @@ } }) if(self.competition.participant_status === 'approved' && self.competition.make_programs_available){ - self.competition.files.push(ingestion_program) - self.competition.files.push(scoring_program) + Object.keys(ingestion_program).length != 0 ? self.competition.files.push(ingestion_program) : null + Object.keys(scoring_program).length != 0 ? self.competition.files.push(scoring_program) : null }if(self.competition.participant_status === 'approved' && self.competition.make_input_data_available){ - self.competition.files.push(input_data) + Object.keys(input_data).length != 0 ? self.competition.files.push(input_data) : null } if(self.competition.admin && !self.competition.make_programs_available){ - self.competition.files.push(ingestion_program) - self.competition.files.push(scoring_program) + Object.keys(ingestion_program).length != 0 ? self.competition.files.push(ingestion_program) : null + Object.keys(scoring_program).length != 0 ? self.competition.files.push(scoring_program) : null } if(self.competition.admin && !self.competition.make_input_data_available){ - self.competition.files.push(input_data) + Object.keys(input_data).length != 0 ? self.competition.files.push(input_data) : null } if(self.competition.admin){ - self.competition.files.push(reference_data) + Object.keys(reference_data).length != 0 ? self.competition.files.push(reference_data) : null } + }) // Need code for public_data and starting_kit at phase level if(self.competition.participant_status === 'approved'){ _.forEach(phase.tasks, task => { _.forEach(task.solutions, solution => { - self.competition.files.push({ + soln = { key: solution.data, name: solution.name, file_size: solution.size, @@ -272,11 +273,12 @@ task: task.name, type: 'Solution', available: true - }) + } + Object.keys(solution).length != 0 ? self.competition.files.push(soln) : null }) }) if (phase.starting_kit != null){ - self.competition.files.push({ + s_kit = { key: phase.starting_kit.key, name: phase.starting_kit.name, file_size: phase.starting_kit.file_size, @@ -284,10 +286,11 @@ task: '-', type: 'Starting Kit', available: true - }) + } + Object.keys(phase.starting_kit).length != 0 ? self.competition.files.push(s_kit) : null } if (phase.public_data != null){ - self.competition.files.push({ + p_data = { key: phase.public_data.key, name: phase.public_data.name, file_size: phase.public_data.file_size, @@ -295,7 +298,8 @@ task: '-', type: 'Public Data', available: true - }) + } + Object.keys(phase.public_data).length != 0 ? self.competition.files.push(p_data) : null } } }) From 9b0d3ad3c9af3e986cabd92e521594bd4827582e Mon Sep 17 00:00:00 2001 From: dtuantran Date: Tue, 19 Sep 2023 16:01:44 +0200 Subject: [PATCH 013/136] cast str to int --- src/apps/api/views/competitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index a68052f5b..095eb7c9b 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -602,7 +602,7 @@ def rerun_submissions(self, request, pk): elif request.user in comp.all_organizers: # submissions are in limit - if len(submissions) <= settings.RERUN_SUBMISSION_LIMIT: + if len(submissions) <= int(settings.RERUN_SUBMISSION_LIMIT): can_re_run_submissions = True # submissions are not in limit From a93810c7ffd2cc89f8ad447f30dcb96be6b538b8 Mon Sep 17 00:00:00 2001 From: dtuantran Date: Wed, 20 Sep 2023 11:26:51 +0200 Subject: [PATCH 014/136] Update docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e4e9e8180..22311bb68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -126,7 +126,7 @@ services: # Rabbitmq & Flower monitoring tool #----------------------------------------------- rabbit: - image: rabbitmq:3.6-management + image: rabbitmq:management # setting hostname here makes data persist properly between # containers being destroyed..! hostname: rabbit From 6927700a04b8f2890b2fd6db19ad88a243027f70 Mon Sep 17 00:00:00 2001 From: acletournel Date: Wed, 20 Sep 2023 17:03:29 +0200 Subject: [PATCH 015/136] First guide lines for contributors --- .github/CONTRIBUTING.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/CONTRIBUTING.md diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 000000000..1acf33d3e --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,23 @@ +# HOW YOU CAN CONTRIBUTE TO THE CODABENCH PROJECT + +## preliminary step: being a codabench user. + +- create a user account on https://codalab.lisn.fr and on https://codabench.org. +- register on https://codabench.org to this existing competition (IRIS-tuto) https://www.codabench.org/competitions/1115/ and make a submission (from https://github.com/codalab/competition-examples/tree/master/codabench/iris): sample_result_submission and sample_code_submission. See https://github.com/codalab/codabench/wiki/User_Participating-in-a-Competition +- create your own private competition (from https://github.com/codalab/competition-examples/tree/master/codabench/ ). See https://github.com/codalab/codabench/wiki/Getting-started-with-Codabench + + ## First step: setting a local instance of Codabench. + +- Follow the tutorial in codabench wiki: https://github.com/codalab/codabench/wiki/Codabench-Installation. According to your hosting OS, you might have to tune your environment file a bit. Try without enabling the SSL protocol (doing so, you don't need a domain name for the server). Try using the embedded Minio storage solution instead of a private cloud storage. +- if needed, you can also looked into https://github.com/codalab/codabench/wiki/How-to-deploy-Codabench-on-your-server + +## Third step: using one's local instance + +- create your own competition and play with it. You can look at the output logs of each different docker container. +- setting you as an admin of your platform (https://github.com/codalab/codabench/wiki/Administrator-procedures#give-superuser-privileges-to-an-user) and visit the Django Admin menu: https://github.com/codalab/codabench/wiki/Administrator-procedures#give-superuser-privileges-to-an-user + +## Fourth step: setting an autonomous computer-worker on your PC + +- configure and launch the docker container: https://github.com/codalab/codabench/wiki/Compute-Worker-Management---Setup +- create a private queue on your new own competition on the production server codabench.org: https://github.com/codalab/codabench/wiki/Queue-Management#create-queue +- assign your own compute-worker to this private queue instead of the default queue. From 134df42f9ee2c1ebc8be1f649270e3b8dd6c75ab Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 22 Sep 2023 19:43:15 +0500 Subject: [PATCH 016/136] clickable report link added --- src/static/img/paper.png | Bin 0 -> 3690 bytes src/static/riot/competitions/public-list.tag | 36 ++++++++++-------- .../competitions/tile/competition_tile.tag | 18 ++++++--- 3 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 src/static/img/paper.png diff --git a/src/static/img/paper.png b/src/static/img/paper.png new file mode 100644 index 0000000000000000000000000000000000000000..8cf0231d8d43f6c174c32ae2fa2c1ff0f6c98264 GIT binary patch literal 3690 zcmb7HbySqy)_-OgkQ%yMkai>lh6ZV*8ze3v2n;bu*Grd_h%`uyAWDcZ^w2d3_|izH zgfvnj$Q{4$y=&cn?^)}dy-x1mjJCT@_?N7dZN$kWNk-|o31;P3DMz{LaM z?O=y=eBk-~S>Ayx6OO}{rphDZfc*V}Kyw;(rv7DuW{5Gx1`FkeAqUtMP^F1iPipDv z4QLgyImYJp_N7yhV-5w0X3I6jhSb(0n5aeUOE_)Hj}oLz|>2~5cp zk* zxlD|}E7^XlKrR&gkSE!2mMr~l_z{yLgaa85KvD!k!`lIZV&7xFDQ0eDR4w=jg4B%J zC6ra2r4Xjr*hJ8LCJtW+fueNaJ}3Dm$g`si%Z7*cbi_Z#=MCi|@aOIU!CN**r-51? zIgf%-4aPdQL?;Y z4Kc4Dnx|W_vja7F#8dQ++zH>SBz3c@%lB=?&?*KiD>E>lx8|UxJ8n*2T6C6T0f%+6 zOZo47?Y9RAV-u6-`(*1rb%!Z^b`|E0LzW<*v$L~{fk84G0T*M!tp>|bfCyO}@f9sA zD^)3PFb90Wcc|aoj*d+!!+FWYTa|5@9r2NDomh}^KX+wvQcSa>8BYjjO}u~}h?l-9 zk?Aq0ZR!9T;Tv$nV1C_1_)eEa)xv^*rdl{95ur!QCIO~V{p6a}2rjRv=sB`gc0Cp?-pwu zQWy-zBE^_kk+rUGb)AjyXNW1`YdS|prE8H-yH{`msqFqSXDoE0P&$coSO0Q;>6Vsl zb_#4cv>4?^cmzhSuPjYHDonaivKZw0KM88Vgq?M!%2p;RlfQ_n@^x2a+=IlmdD1_O z(SqrZ&iTVT$^MvzP;uqRj|BkE4}Z}us~$wZZOKKLF$}0p(lR%O;;Hk}o9>^O0^EJ6 zp8~q9G)lX-ot+%Z984}buPJDIP&h1=n9-8F(D-Od(k;R%vC7p7;Pz zpsjuU`>Uo^Yw_5T1cc&i#6NI2fe7XQzc2so)BnSym<_Pw!jIL^`!)w%Dj5Syh8*!; z21@nJOc0^eA$Pf@5)fGdacU&Cx9y(HVFLvUah-p))j0JTYTV1cElDVGMgulW&L!0i zbo20!F@;$V$<%JJX=c|-D13c0OF?)ChXQ%w@D(plDr-#sp0;1X8e(J-RxD z{~_$8X}d3$Zf~ewu=f;Vy z75^>U!LsGmrM%zr?GX`_h~LNf{Se!k#pKL(pH;mw?LqEUCTpN!F=%|eY_lPok|}|) zI1P#1fx3q#Ll=f^?VLC+m~e8RAZMS{M7V8 z#NBR@HCE_M^GS)JRh+jO3G&+^=j0?7xtDcow>jx``E3i&*?F{bI-+?%@r6=H=gWQ{ zYj@arNNXa$w^+&fVQFb_0GaXoH%XYsA#H~=PI`s$+dT>n2T6kr;ywV?;gV@w9wvM& zqUi3SE~@lGR%Hu73__!_x4k<)+Y!la>#JBg%oWP%+RIwi_V4f!`m@*_`9MC<3$ z9+zI)4?aW7TFc>G;-`l8(Y{bNpt&N5w%yrexFkq%X~;5I_~}h}a?yP?wHm;ghMr~1 zs}W@s6ZDah&dr9GJsdX@dGM4$Kw$C$x&p1}$@Bd}&@M;Is&jwRsb};nWc^uH&o%8G zyTy#(ltXj5Ol}q(#31>DORmk=b8ezgLGbqEEciA>QZbLC#p{N+KZevuON%0OgTv3m zT5o7~bdaSB<G5sxF$J zx_!mP#8woR_d4UKm2!2y5hQ1m&DelkkDJA%bDymA{FD)XHcKwtxThvqQBC(J21Cv+ zdP@_-KCPwJ*0ggxy-pH3IxZu0?DWT_az zWBPAogXiqBAP4y;pciX?#YV>{mg`ziDJsw3vz3(&1G?r^?Mz1Uxl?Pp#cND`ZvUhy zuDNHPKpyQ`#zPPg-c(B(+$hw$;}ASLdM{QmF~aUnQUTjtIGy+4K0-wD`sVR#S~z|4 zJ2UpmavbHP24H{oQm& zHI=R8qn^U{?!+^;-KQ^Ru^L+a30wv`b@l+C?0CNm$&qL(`zGi4%uN$bFyy@^+}MMb z??vSMW>}@lgPMb@(;BMgqQne2wB*+!Mu6Dc2LMt=K{BNVM)*U*6olkFDb_L@Z$n6$!$F4;h%hhed~)I(N^(r;)E?1wTb*;gZ6cmb!$B{c7;Z8 z%CZ~!9kM1ZlezcAWaE{83>%vs7${J$>7XyuEsP3-y$CO+^j>F3mgf(YN5H~GjS~f5 z78WQ`t2upkf1M%gdb38iJ^CJpUbhDA6^-p$e=n(`J32OIl(^iSlOT!aWy#@(=Ve%! zKj9K{gQ=yu2VVFjLqXq*-tw`}?j+z4U!dzXI&iSz`j)7x=;Q`JnQTknsM4y3_uX0V z*80a~U>R{Q#t2UWO>uEC+l-LP`u6WB3bRZJ!3$=J=!{)|RBQS*6fGtJDaSMNMr7w& z_DpQ`<#tHKQRdLZG{5H?F75Dg8(m^>%JjowjKp?XuL>&}JlSTZ0K*Rz^W*2@FTWPL zIItYk+lz$g3u*tld^r5CCYGR}E7)Y2EYl*GFG|HK0kik?OOM8radDP8>XgX*4JSwI zR^NdWUl|d;%l&>?DlkLYhY}i#or{;b*+mV)C8br#4@}RK?Rvjk3`}(QQ_`pw+wkCY zd~7uIBch;w63y+-UtZ_umF;Fw_ zdnw=djG-|ZpS@_}t6bu7Fft~hSj1}i2yt$hgb{v~=|e59NApWxRqK$`813~4N(qn% zQ>snkQv&MFh{mYWWW4+CEY5buoYg9(S8~xKY^0jLCM4n*s?tn$O3)E0p0Yzz&}F-9 zULbSlJg@-5fI8P_J>h^(Auj@yZ`995*}1FyyN;_&(?$l+4GYGM<#Vlp%n5j~y|Hxy3W~?XP>q5P-z}3js}4U6pDj+sJS literal 0 HcmV?d00001 diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index 3b51ddbc9..8a3e09683 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -8,30 +8,33 @@
@@ -142,7 +145,10 @@ margin auto .link-no-deco + all unset text-decoration none + cursor pointer + width 100% .tile-wrapper border solid 1px gainsboro diff --git a/src/static/riot/competitions/tile/competition_tile.tag b/src/static/riot/competitions/tile/competition_tile.tag index 5d2dbbf7b..5c4fdcb7e 100644 --- a/src/static/riot/competitions/tile/competition_tile.tag +++ b/src/static/riot/competitions/tile/competition_tile.tag @@ -1,9 +1,10 @@ - +
+

{title} @@ -15,14 +16,17 @@ Organized by: {created_by}

-
+ +
{pretty_date(created_when)} -
-
+
+
+ + +
{participant_count} Participants
- + \ No newline at end of file diff --git a/src/static/riot/analytics/_competitions_usage.tag b/src/static/riot/analytics/_competitions_usage.tag new file mode 100644 index 000000000..aef3ab47d --- /dev/null +++ b/src/static/riot/analytics/_competitions_usage.tag @@ -0,0 +1,480 @@ + + + +
+ +
+
+
+ + +
+
+
+ +
+ + + + + + + + + + + + + + + + + +
CompetitionOrganizerCreation dateDatasets
{ competitionUsage.title }{ competitionUsage.organizer }{ competitionUsage.created_when }{ competitionUsage.datasets }
+ + + + + \ No newline at end of file diff --git a/src/static/riot/analytics/_usage_history.tag b/src/static/riot/analytics/_usage_history.tag new file mode 100644 index 000000000..efd90cd42 --- /dev/null +++ b/src/static/riot/analytics/_usage_history.tag @@ -0,0 +1,151 @@ + +
+ +
+ + + + + \ No newline at end of file diff --git a/src/static/riot/analytics/_users_usage.tag b/src/static/riot/analytics/_users_usage.tag new file mode 100644 index 000000000..3f6ee7952 --- /dev/null +++ b/src/static/riot/analytics/_users_usage.tag @@ -0,0 +1,480 @@ + + + +
+ +
+
+
+ + +
+
+
+ +
+ + + + + + + + + + + + + + + + + +
UserJoined atDatasetsSubmissions
{ userUsage.name }{ userUsage.date_joined }{ userUsage.datasets }{ userUsage.submissions }
+ + + + + \ No newline at end of file diff --git a/src/static/riot/analytics/analytics.tag b/src/static/riot/analytics/analytics.tag index a19431a78..905016a06 100644 --- a/src/static/riot/analytics/analytics.tag +++ b/src/static/riot/analytics/analytics.tag @@ -136,63 +136,19 @@
-
- -
+
- - -
- -
-
-
- - -
-
-
- -
- - - - - - - - - - - - - - - - - - - - - - -
CompetitionOrganizerCreation dateDatasets
{ competitionUsage.title }{ competitionUsage.organizer }{ competitionUsage.created_when }{ competitionUsage.datasets }
+
-

TODO

+
-

TODO

+
@@ -202,7 +158,7 @@ /*--------------------------------------------------------------------- Init ---------------------------------------------------------------------*/ - + self.current_view = "overview"; self.currentAnalyticsTab = "overview"; self.currentStorageTab = "usageHistory"; self.isDataReloadNeeded = { @@ -219,6 +175,8 @@ let datetime = luxon.DateTime; self.start_date = datetime.local(datetime.local().year); self.end_date = datetime.local(); + self.start_date_string = self.start_date.toISODate(); + self.end_date_string = self.end_date.toISODate(); self.colors = ["#36a2eb", "#ff6384", "#4bc0c0", "#ff9f40", "#9966ff", "#ffcd56", "#c9cbcf"]; @@ -234,18 +192,8 @@ /****** Storage *****/ // Usage history - self.storageUsageHistoryData = null; - self.competitionsUsageData = null; self.usersUsageData = null; self.adminUsageData = null; - self.storageUsageChart; - - // Competitions usage - self.competitionsDropdownOptions = []; - self.storageCompetitionsUsageChart; - self.storageCompetitionsUsagePieChart; - self.competitionsUsageTableData = []; - self.competitionsColor = {}; // Users usage @@ -294,6 +242,8 @@ endCalendar: $(self.refs.end_calendar), onChange: function(date, text) { self.start_date = datetime.fromJSDate(date) + self.start_date_string = self.start_date.toISODate(); + self.update({start_date_string: self.start_date_string}); let end_date = $(self.refs.end_calendar).calendar('get date') if (!!end_date && date > end_date) { @@ -304,16 +254,6 @@ Object.keys(self.isDataReloadNeeded["storage"]).forEach(v => self.isDataReloadNeeded["storage"][v] = true); if (self.currentAnalyticsTab == "overview") { self.update_analytics(self.start_date, self.end_date, self.time_unit); - } else { - if (self.currentStorageTab == "usage-history") { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "competitions-usage") { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "users-usage") { - console.log("TODO"); - } else if (self.currentStorageTab == "admin-usage") { - console.log("TODO"); - } } } } @@ -324,22 +264,14 @@ onChange: function(date, text) { if (date) { self.end_date = datetime.fromJSDate(date) + self.end_date_string = self.end_date.toISODate(); + self.update({end_date_string: self.end_date_string}); } self.isDataReloadNeeded["overview"] = true; Object.keys(self.isDataReloadNeeded["storage"]).forEach(v => self.isDataReloadNeeded["storage"][v] = true); if (self.currentAnalyticsTab == "overview") { self.update_analytics(self.start_date, self.end_date, self.time_unit); - } else { - if (self.currentStorageTab == "usage-history") { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "competitions-usage") { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "users-usage") { - console.log("TODO"); - } else if (self.currentStorageTab == "admin-usage") { - console.log("TODO"); - } } }, } @@ -369,205 +301,6 @@ $('.top.menu.analytics .item').tab('change tab', 'overview'); $('.storage .top.menu .item').tab('change tab', 'usage-history'); - /*--------------------------------------------------------------------- - Storage Usage History Chart Setup - ---------------------------------------------------------------------*/ - let storageUsageConfig = { - type: 'line', - data: { - datasets: [ - { - label: 'Total usage', - data: [], - borderColor: 'rgb(255, 99, 132)', - borderWidth: 1, - lineTension: 0 - }, - { - label: 'Competitions usage', - data: [], - borderColor: 'rgb(255, 164, 74)', - borderWidth: 1, - lineTension: 0 - }, - { - label: 'Users usage', - data: [], - borderColor: 'rgb(54, 162, 235)', - borderWidth: 1, - lineTension: 0 - }, - { - label: 'Administration usage', - data: [], - borderColor: 'rgb(153, 102, 255)', - borderWidth: 1, - lineTension: 0 - }, - { - label: 'Orphaned files usage', - data: [], - borderColor: 'rgb(228, 229, 231)', - borderWidth: 1, - lineTension: 0 - } - ], - }, - options: { - responsive: true, - interaction: { - mode: 'nearest', - axis: 'x', - intersect: false - }, - scales: { - xAxes: [{ - type: 'time', - ticks: { - source: 'auto' - } - }], - yAxes: [{ - ticks: { - beginAtZero: true, - stepSize: 'auto', - callback: function(value, index, values) { - return pretty_bytes(value); - } - } - }] - }, - tooltips: { - mode: 'index', - intersect: false, - position: 'nearest', - callbacks: { - label: function(tooltipItem, data) { - return pretty_bytes(tooltipItem.yLabel); - } - } - } - } - }; - self.storageUsageChart = new Chart($(self.refs.storage_usage_history_chart), storageUsageConfig); - - /*--------------------------------------------------------------------- - Competition usage - ---------------------------------------------------------------------*/ - - // Sementic UI components setups - self.selectedCompetitions = []; - $(self.refs.competitions_dropdown).dropdown({ - onAdd: self.addCompetitionToSelection, - onRemove: self.removeCompetitionFromSelection, - clearable: true, - preserveHTML: false, - - }); - $('#storageCompetitionsTable').tablesort(); - self.tableSelectedDate = null; - let table_date_specific_options = { - onChange: function(date, text) { - self.tableSelectedDate = date; - self.updateCompetitionsTable(); - self.updateCompetitionsChart(); - self.updateCompetitionsPieChart(); - } - }; - let table_date_calendar_options = _.assign({}, general_calendar_options, table_date_specific_options); - $(self.refs.table_date_calendar).calendar(table_date_calendar_options); - - /*--------------------------------------------------------------------- - Competitions Usage Chart Setup - ---------------------------------------------------------------------*/ - - let storageCompetitionsUsageConfig = { - type: 'line', - data: { - datasets: [], - }, - options: { - responsive: true, - interaction: { - mode: 'nearest', - axis: 'x', - intersect: false - }, - scales: { - xAxes: [{ - type: 'time', - ticks: { - source: 'auto' - } - }], - yAxes: [{ - ticks: { - beginAtZero: true, - stepSize: 'auto', - callback: function(value, index, values) { - return pretty_bytes(value); - } - } - }] - }, - tooltips: { - mode: 'index', - intersect: false, - position: 'nearest', - callbacks: { - label: function(tooltipItem, data) { - return pretty_bytes(tooltipItem.yLabel); - } - } - } - } - }; - - self.storageCompetitionsUsageChart = new Chart($(self.refs.storage_competitions_usage_chart), storageCompetitionsUsageConfig); - - /*--------------------------------------------------------------------- - Competitions Usage Pie Chart Setup - ---------------------------------------------------------------------*/ - - let storageCompetitionsUsagePieConfig = { - type: 'pie', - data: { - labels: [], - competitionsId: [], - datasets: [ - { - label: 'Competitions distribution', - backgroundColor: [], - hoverOffset: 4, - data: [] - } - ], - }, - options: { - responsive: true, - plugins: { - legend: { - position: 'left', - }, - title: { - display: true, - text: 'Competitions distribution' - } - }, - tooltips: { - callbacks: { - label: function(tooltipItem, data) { - const label = data.labels[tooltipItem.index]; - const value = pretty_bytes(data.datasets[0].data[tooltipItem.index]); - return " " + label + ": " + value; - } - } - } - } - }; - - self.storageCompetitionsUsagePieChart = new Chart($(self.refs.storage_competitions_usage_pie), storageCompetitionsUsagePieConfig); - /*--------------------------------------------------------------------- Initialization ---------------------------------------------------------------------*/ @@ -636,23 +369,6 @@ chart.update() } - self.update_storage_usage_history_chart = function(data) { - var list_usages = {}; - for (let [date, usages] of Object.entries(data)) { - for (let [usage_label, usage] of Object.entries(usages)) { - if (!list_usages.hasOwnProperty(usage_label)) { - list_usages[usage_label] = []; - } - list_usages[usage_label].push({x: new Date(date), y: usage * 1024}); - } - } - for (const [index, usage_label] of Object.entries(Object.keys(list_usages))) { - list_usages[usage_label].sort(function(a, b) {return a.x - b.x;}); - self.storageUsageChart.data.datasets[index].data = list_usages[usage_label]; - } - self.storageUsageChart.update(); - } - self.update_analytics = function (start, end, time_unit) { if (!end) { end = datetime.local() @@ -695,73 +411,27 @@ self.onAnalyticsTabChange = function (tabName) { self.currentAnalyticsTab = tabName; + if (tabName == "overview") { + self.current_view = self.currentAnalyticsTab; + self.update({current_view: self.currentAnalyticsTab}); + } else if (tabName == "storage") { + self.current_view = self.currentStorageTab; + self.update({current_view: self.currentStorageTab}); + } + if (tabName == "overview" && self.isDataReloadNeeded["overview"]) { self.update_analytics(self.start_date, self.end_date, self.time_unit); - } else if (tabName == "storage") { - if (self.currentStorageTab == "usage-history" && self.isDataReloadNeeded["storage"]["usageHistory"]) { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "competitions-usage" && self.isDataReloadNeeded["storage"]["competitionsUsage"]) { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "users-usage" && self.isDataReloadNeeded["storage"]["usersUsage"]) { - console.log("TODO"); - } else if (self.currentStorageTab == "admin-usage" && self.isDataReloadNeeded["storage"]["adminUsage"]) { - console.log("TODO"); - } } }; self.onStorageTabChange = function(tabName) { self.currentStorageTab = tabName; - if (tabName == "usage-history" && self.isDataReloadNeeded["storage"]["usageHistory"]) { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (tabName == "competitions-usage" && self.isDataReloadNeeded["storage"]["competitionsUsage"]) { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (tabName == "users-usage" && self.isDataReloadNeeded["storage"]["usersUsage"]) { - console.log("TODO"); - } else if (tabName == "admin-usage" && self.isDataReloadNeeded["storage"]["adminUsage"]) { - console.log("TODO"); + if (self.current_view != "overview") { + self.current_view = self.currentStorageTab; + self.update({current_view: self.currentStorageTab}); } } - self.get_storage_usage_history = function(start_date, end_date, resolution) { - let parameters = { - start_date: start_date.toISODate(), - end_date: end_date.toISODate(), - resolution: resolution - }; - CODALAB.api.get_storage_usage_history(parameters) - .done(function(data) { - self.update({storageUsageHistoryData: data}); - self.update_storage_usage_history_chart(data); - self.isDataReloadNeeded["storage"]["usageHistory"] = false; - }) - .fail(function(error) { - toastr.error("Could not load storage analytics data"); - }); - } - - self.get_competitions_usage = function(start_date, end_date, resolution) { - let parameters = { - start_date: start_date.toISODate(), - end_date: end_date.toISODate(), - resolution: resolution - }; - CODALAB.api.get_competitions_usage(parameters) - .done(function(data) { - self.competitionsUsageData = data; - self.update({competitionsUsageData: data}); - self.updateCompetitionsSelectionDropdown(); - self.updateCompetitionTableCalendar(data); - self.updateCompetitionsChart(); - self.updateCompetitionsPieChart(); - self.updateCompetitionsTable(); - self.isDataReloadNeeded["storage"]["competitionsUsage"] = false; - }) - .fail(function(error) { - toastr.error("Could not load storage analytics data"); - }); - } - // Shortcut buttons self.time_range_shortcut = function(unit_selection) { self.end_date = datetime.local(); @@ -776,6 +446,9 @@ self.shortcut_dropdown.dropdown('set selected', unit_selection); self.time_unit = 'day'; + self.start_date_string = self.start_date.toISODate(); + self.end_date_string = self.end_date.toISODate(); + if (unit_selection !== 'year') { self.resolution_dropdown.dropdown('set selected', 'day'); } else { @@ -783,27 +456,20 @@ self.resolution_dropdown.dropdown('set selected', 'month'); } + self.update({start_date_string: self.start_date_string, end_date_string: self.end_date_string, time_unit: self.time_unit}); + self.isDataReloadNeeded["overview"] = true; Object.keys(self.isDataReloadNeeded["storage"]).forEach(v => self.isDataReloadNeeded["storage"][v] = true); if (self.currentAnalyticsTab == "overview") { self.update_analytics(self.start_date, self.end_date, self.time_unit); - } else { - if (self.currentStorageTab == "usage-history") { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "competitions-usage") { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "users-usage") { - console.log("TODO"); - } else if (self.currentStorageTab == "admin-usage") { - console.log("TODO"); - } } } // Chart Units (Months, Weeks, Days) self.update_chart_resolution = function(unit_selection) { self.time_unit = unit_selection; + self.update({time_unit: self.time_unit}); self.resolution_dropdown.dropdown('set selected', unit_selection); self.competitionsChart.options.scales.xAxes[0].time.unit = unit_selection; @@ -818,254 +484,10 @@ if (self.currentAnalyticsTab == "overview") { self.update_analytics(self.start_date, self.end_date, self.time_unit); - } else { - if (self.currentStorageTab == "usage-history") { - self.get_storage_usage_history(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "competitions-usage") { - self.get_competitions_usage(self.start_date, self.end_date, self.time_unit); - } else if (self.currentStorageTab == "users-usage") { - console.log("TODO"); - } else if (self.currentStorageTab == "admin-usage") { - console.log("TODO"); - } } self.update(); } - self.addCompetitionToSelection = function(value, text, $addedItem) { - if(Object.keys(self.competitionsUsageData).length > 0) { - self.selectedCompetitions.push(value); - let competitionUsage = []; - for (let [dateString, competitions] of Object.entries(self.competitionsUsageData)) { - for (let [competitionId, competition] of Object.entries(competitions)) { - if (competitionId == value) { - competitionUsage.push({x: new Date(dateString), y: competition.datasets * 1024}); - } - } - } - const competitions = Object.values(self.competitionsUsageData)[0]; - const competitionTitle = competitions[value].title; - if(!self.competitionsColor.hasOwnProperty(value)) { - self.competitionsColor[value] = self.colors[Object.keys(self.competitionsColor).length % self.colors.length]; - } - const color = self.competitionsColor[value]; - - // Update chart - self.storageCompetitionsUsageChart.data.datasets.push({ - competitionId: value, - label: competitionTitle, - data: competitionUsage, - backgroundColor: color, - borderWidth: 1, - lineTension: 0, - fill: false - }); - self.storageCompetitionsUsageChart.update(); - - // Update pie chart - let selectedDate = self.tableSelectedDate; - if (!selectedDate) { - selectedDate = new Date(Object.keys(self.competitionsUsageData).reduce((acc, cur) => new Date(acc) > new Date(cur) ? acc : cur , '0000-00-00')); - } - const selectedDateString = selectedDate.getUTCFullYear() + "-" + (selectedDate.getUTCMonth()+1) + "-" + selectedDate.getUTCDate(); - const closestOlderDateString = Object.keys(self.competitionsUsageData).reduce((acc, cur) => (Math.abs(new Date(selectedDateString) - new Date(cur)) < Math.abs(new Date(selectedDateString) - new Date(acc)) && (new Date(selectedDateString) - new Date(cur) >= 0)) ? cur : acc, '9999-12-31'); - const competitionsAtSelectedDate = self.competitionsUsageData[closestOlderDateString]; - const selectedCompetitions = Object.keys(competitionsAtSelectedDate).filter(date => self.selectedCompetitions.includes(date)).reduce((competition, date) => ({ ...competition, [date]: competitionsAtSelectedDate[date] }), {}); - - const {labels, competitionsId, data} = self.formatDataForCompetitionsPieChart(selectedCompetitions); - self.storageCompetitionsUsagePieChart.data.labels = labels; - self.storageCompetitionsUsagePieChart.data.competitionsId = competitionsId; - self.storageCompetitionsUsagePieChart.data.datasets[0].data = data; - self.storageCompetitionsUsagePieChart.data.datasets[0].labels = labels; - self.storageCompetitionsUsagePieChart.data.datasets[0].backgroundColor = self.listOfColors(data.length); - self.storageCompetitionsUsagePieChart.update(); - } - } - - self.listOfColors = function(arrayLength) { - return Array.apply(null, Array(arrayLength)).map(function (x, i) { return self.colors[i%self.colors.length]; }) - } - - self.formatDataForCompetitionsPieChart = function (data) { - var labels = []; - var competitionsId = []; - var formattedData = []; - - const competitionArray = Object.entries(data).map(([key, value]) => ({ ...value, id: key })); - competitionArray.sort((a, b) => b.datasets - a.datasets); - for (const competition of competitionArray) { - labels.push(competition.title); - competitionsId.push(competition.id); - formattedData.push(competition.datasets * 1024); - } - - return {labels: labels, competitionsId: competitionsId, data: formattedData}; - } - - self.removeCompetitionFromSelection = function(value, text, $removedItem) { - // Remove from selection - const indexToRemoveInSelected = self.selectedCompetitions.findIndex(competitionId => competitionId == value); - if (indexToRemoveInSelected !== -1) { - self.selectedCompetitions.splice(indexToRemoveInSelected, 1); - } - - // Reassign competitions color - self.competitionsColor = {}; - for(const competitionId of self.selectedCompetitions) { - self.competitionsColor[competitionId] = self.colors[Object.keys(self.competitionsColor).length % self.colors.length]; - } - - // Remove from competition usage chart - let indexToRemove = self.storageCompetitionsUsageChart.data.datasets.findIndex(dataset => dataset.competitionId == value); - if (indexToRemove !== -1) { - self.storageCompetitionsUsageChart.data.datasets.splice(indexToRemove, 1); - for(let dataset of self.storageCompetitionsUsageChart.data.datasets) { - dataset.backgroundColor = self.competitionsColor[dataset.competitionId] - } - self.storageCompetitionsUsageChart.update(); - } - - // Remove from competition pie chart - indexToRemove = self.storageCompetitionsUsagePieChart.data.competitionsId.findIndex(id => id == value); - if (indexToRemove !== -1) { - self.storageCompetitionsUsagePieChart.data.labels.splice(indexToRemove, 1); - self.storageCompetitionsUsagePieChart.data.competitionsId.splice(indexToRemove, 1); - self.storageCompetitionsUsagePieChart.data.datasets[0].data.splice(indexToRemove, 1); - self.storageCompetitionsUsagePieChart.data.datasets[0].backgroundColor.splice(indexToRemove, 1); - self.storageCompetitionsUsagePieChart.data.datasets[0].backgroundColor = self.storageCompetitionsUsagePieChart.data.competitionsId.map(competitionId => self.competitionsColor[competitionId]); - self.storageCompetitionsUsagePieChart.update(); - } - } - - self.updateCompetitionsSelectionDropdown = function () { - // Remove current selection - $(self.refs.competitions_dropdown).dropdown('clear'); - - // Update the options - let competitionsOptions = []; - if(Object.keys(self.competitionsUsageData).length > 0) { - const competitions = Object.values(self.competitionsUsageData)[0]; - competitionsOptions = Object.entries(competitions).map(([id, { title }]) => ({ id, title })); - } - self.competitionsDropdownOptions = competitionsOptions; - $(self.refs.competitions_dropdown).dropdown('change values', competitionsOptions); - self.update({competitionsDropdownOptions: competitionsOptions}); - } - - self.selectTopFiveBiggestCompetitions = function () { - let selectCompetitions = []; - if (Object.keys(self.competitionsUsageData).length > 0) { - const mostRecentDateString = Object.keys(self.competitionsUsageData).reduce((acc, cur) => new Date(acc) > new Date(cur) ? acc : cur ); - let competitions = Object.entries(self.competitionsUsageData[mostRecentDateString]); - competitions.sort((a, b) => b[1].datasets - a[1].datasets); - selectCompetitions = competitions.slice(0, 5).map(([id]) => id); - } - self.selectedCompetitions = selectCompetitions; - $(self.refs.competitions_dropdown).dropdown('clear'); - for(const competitionId of selectCompetitions) { - $(self.refs.competitions_dropdown).dropdown('set selected', competitionId); - } - } - - self.updateCompetitionTableCalendar = function(data) { - // Set the min and max date of the calendar - const minDate = new Date(Object.keys(data).reduce((acc, cur) => new Date(acc) < new Date(cur) ? acc : cur, '9999-12-31')); - const maxDate = new Date(Object.keys(data).reduce((acc, cur) => new Date(acc) > new Date(cur) ? acc : cur, '0000-00-00')); - $(self.refs.table_date_calendar).calendar('setting', 'minDate', minDate); - $(self.refs.table_date_calendar).calendar('setting', 'maxDate', maxDate); - - // Select the most current date available - self.tableSelectedDate = maxDate; - $(self.refs.table_date_calendar).calendar('set date', maxDate); - $(self.refs.table_date_calendar).calendar('refresh'); - } - - self.updateCompetitionsChart = function() { - if(Object.keys(self.competitionsUsageData).length > 0) { - const selectedCompetitions = Object.fromEntries( - Object.entries(self.competitionsUsageData).map(([dateString, competitions]) => [ - dateString, - Object.fromEntries( - Object.entries(competitions).filter(([competitionId]) => self.selectedCompetitions.includes(competitionId)) - ) - ]) - ); - - const competitionsUsage = {}; - for (let [dateString, competitions] of Object.entries(selectedCompetitions)) { - for (let [competitionId, competition] of Object.entries(competitions)) { - if (!competitionsUsage.hasOwnProperty(competitionId)) { - competitionsUsage[competitionId] = []; - } - competitionsUsage[competitionId].push({x: new Date(dateString), y: competition.datasets * 1024}); - } - } - - self.storageCompetitionsUsageChart.data.datasets = []; - let index = 0; - for(let [competitionId, dataset] of Object.entries(competitionsUsage)) { - const color = self.colors[index % self.colors.length]; - const title = Object.values(self.competitionsUsageData)[0][competitionId].title; - self.storageCompetitionsUsageChart.data.datasets.push({ - competitionId: competitionId, - label: title, - data: dataset, - backgroundColor: color, - borderWidth: 1, - lineTension: 0, - fill: false - }); - index++; - } - - self.storageCompetitionsUsageChart.update(); - } - } - - self.updateCompetitionsPieChart = function() { - let selectedDate = self.tableSelectedDate; - if (!selectedDate) { - selectedDate = new Date(Object.keys(self.competitionsUsageData).reduce((acc, cur) => new Date(acc) > new Date(cur) ? acc : cur , '0000-00-00')); - } - const selectedDateString = selectedDate.getUTCFullYear() + "-" + (selectedDate.getUTCMonth()+1) + "-" + selectedDate.getUTCDate(); - const closestOlderDateString = Object.keys(self.competitionsUsageData).reduce((acc, cur) => (Math.abs(new Date(selectedDateString) - new Date(cur)) < Math.abs(new Date(selectedDateString) - new Date(acc)) && (new Date(selectedDateString) - new Date(cur) >= 0)) ? cur : acc, '9999-12-31'); - const competitionsAtSelectedDate = self.competitionsUsageData[closestOlderDateString]; - const selectedCompetitions = Object.keys(competitionsAtSelectedDate).filter(date => self.selectedCompetitions.includes(date)).reduce((competition, date) => ({ ...competition, [date]: competitionsAtSelectedDate[date] }), {}); - - const {labels, competitionsId, data} = self.formatDataForCompetitionsPieChart(selectedCompetitions); - self.storageCompetitionsUsagePieChart.data.labels = labels; - self.storageCompetitionsUsagePieChart.data.competitionsId = competitionsId; - self.storageCompetitionsUsagePieChart.data.datasets[0].data = data; - self.storageCompetitionsUsagePieChart.data.datasets[0].labels = labels; - self.storageCompetitionsUsagePieChart.data.datasets[0].backgroundColor = self.listOfColors(data.length); - self.storageCompetitionsUsagePieChart.update(); - } - - self.updateCompetitionsTable = function() { - const data = self.competitionsUsageData; - let competitionsUsageTableData = []; - if (Object.keys(data).length > 0) { - let selectedDate = self.tableSelectedDate; - if (!selectedDate) { - selectedDate = new Date(Object.keys(data).reduce((acc, cur) => new Date(acc) > new Date(cur) ? acc : cur , '0000-00-00')); - } - const selectedDateString = selectedDate.getUTCFullYear() + "-" + (selectedDate.getUTCMonth()+1) + "-" + selectedDate.getUTCDate(); - const closestOlderDateString = Object.keys(data).reduce((acc, cur) => (Math.abs(new Date(selectedDateString) - new Date(cur)) < Math.abs(new Date(selectedDateString) - new Date(acc)) && (new Date(selectedDateString) - new Date(cur) >= 0)) ? cur : acc, '9999-12-31'); - const competitions = data[closestOlderDateString]; - Object.entries(competitions).forEach(keyValue => { - const [competitionId, competition] = keyValue; - competitionsUsageTableData.push({ - 'id': competitionId, - 'title': competition.title, - 'organizer': competition.organizer, - 'created_when': new Date(competition.created_when).toDateString(), - 'datasets': pretty_bytes(competition.datasets * 1024) - }); - }); - self.update({competitionsUsageTableData: competitionsUsageTableData}); - } - } - - \ No newline at end of file diff --git a/src/static/riot/analytics/_competitions_usage.tag b/src/static/riot/analytics/_competitions_usage.tag index 8c6403e47..b9568313e 100644 --- a/src/static/riot/analytics/_competitions_usage.tag +++ b/src/static/riot/analytics/_competitions_usage.tag @@ -7,6 +7,9 @@ +
@@ -19,6 +22,9 @@
+ @@ -465,6 +471,54 @@ self.formatSize = function(size) { return pretty_bytes(size); } + + self.downloadCompetitionsHistory = function() { + var csv = []; + + // Categories + const competitions = Object.values(self.competitionsUsageData)[0]; + const competitions_id = Object.entries(competitions).map(([id, { title }]) => (id)); + const competitions_name = Object.entries(competitions).map(([id, { title }]) => (title)); + csv.push("," + competitions_id.join(",")); + csv.push("," + competitions_name.join(",")); + + // Data points + sorted_dates = Object.keys(self.competitionsUsageData).sort(function(a, b) {return new Date(a) - new Date(b)}); + for (const date of sorted_dates) { + let points = [date]; + for (const id of competitions_id) { + points.push(self.competitionsUsageData[date][id]['datasets'] * 1024); + } + csv.push(points.join(",")); + } + + // Save + const blob = new Blob([csv.join('\n')], { type: 'text/csv;charset=utf-8;' }); + saveAs(blob, "competitions_usage_history.csv"); + } + + self.downloadCompetitionsTable = function() { + var csv = []; + + // Categories + let categories = ['Competition', 'Organizer', 'Creation date', 'Datasets']; + csv.push(categories.join(",")); + + // Data points + for (const competition of self.competitionsUsageTableData) { + const points = [ + competition.title, + competition.organizer, + competition.created_when.toLocaleString(), + competition.datasets * 1024 + ]; + csv.push(points.join(",")); + } + + // Save + const blob = new Blob([csv.join('\n')], { type: 'text/csv;charset=utf-8;' }); + saveAs(blob, "competitions_table.csv"); + } + + + + + + + + + + diff --git a/src/static/img/lisn-logo.png b/src/static/img/lisn-logo.png deleted file mode 100644 index df72f28212a26849cb97842843b3700486da123b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4218 zcmai&Sv-^t1I6{CX6!TeC9jOJ$JohaX=Y^0zEeyXLX0FTOY&r2vrTqU5;0{TvOHvv zwGyE+WY3<)ny>HryZFw<`JdmpI@c%O!t63Tix3L~0|UE}A=Zk4;S~OFPK7Z3brFHw z(!T&KOl@#~d2(`6RaIqbYD%F{Dk>_Jm6hGx+>VZpc6N5Wyu9didP76Q`1p8eXlQ$T zyT8Bx!otGv@bKE&T3lQll}aU%NP&TY*4EZKIyw;%5t^Esb8~ZIVq)_0@@8gc#l^+x z>FEv*4iXX)TwGi^IXS|@!UhHgA|fKJtgMNNiHeGf>(LUT3=EvFjj;MQq4{eC?DwCx z!eh9x%$?%@MATfbub=!oacDq758mlh`Jqi}ICxDVCM{JMqLR^}Oz`~w+8tMQ6@pU7 z?IyRrk0qHW^R(T3cJ9P6STkgoXDD_}bzWCiY#_s2jTzqR=seh5`Z>?IX6N^06rklQ zEt8qn_$gm|rCTmJb*&!5wcwZ~4!2uVTQ)P6>qx8#(lvXGqQ5_U;ceGGNcgIcqOU1BX%sqgN1F#kPq-yKCKBXMkTBX< z+Zw?e&?-U|kZ=4gp4mUaw)`>4ZDTVXw(~9=ldR23=$+6&ov|-K{kt9vqAO|+m;2x9 zjqzSCOK@x-SiRNS&R&^PO<7bAN|jorFviNBb9XJOx&3iOjBV~=y2Vt(Vr6CHN&=!G z%a<}Uy8mZrVoApJ$%1w2xbwXns4#(K6zTylB)ckEO`A1Q4&G2k6_od%#!&FFz}-Q^ zEnfT}TgyW-f8A0LAEkg;(mXs2t%<(SQOvK#53_T5bjcOIu+^?iV*&L&QaxW8FN zL~m9{De`k3Ub^^%U?AKsSw+H+!xa4HfB(7)!ejI&0wk+ulHZkdIaF*H0hmS%md5V3fnrSI{eg8U=hQFOmc9((i|^7w17+oFbCR z3%|D4S4)cc!#^dvMiajruR?>`-mMDvxmTbC{M>@V>t&_CV;to>g+A>#8piOYFPBo*+@VI$YI zfW4TTXf^junHchE@I0)7Y7|0U5p6dN*9uAXg;>)A#HY`x_*Hpe5$^3ilG9uP z_sITY(-m8BYT&(IDesigaahm#cVmIIv{%Lg)I)z2eAnH`lxit-Mb}f<&=o=k{ott& zw9BxhKzxqm1bW8~^cPG1IgM~9O2d5=;&a_*qJ!0Fw7b4nz@44r%E?D&>_Bt*A5@AA zv=)o3%EgQ8SG-nW3;t~pT30RM!nl!7##bA$50?+*LR@`}aSje_%=mX~8Bkktsm_R( zfO_iZEZ}neO*Lp9O`)RW=SNPcweM$J&qsoGpY55nSwPlf=+-oGDhoJc4TJLNhmu&O{&cTA-^ zwPWWw_jR5FB(3)Dg?!t6Q`Kyru-)AHbM%ox;OZ*`yBwUazAJD!>;|w?OZ@N^QkqVC z34AfB-*Eih{R$}m$qL@`?0!Q)E3842iS|{jp%A=Fl=fIp^i^2lyWE*?^H_V2A<@so(x0AuWSh-HMmnu zK_?#G;mY+5H`mrF>0_)0SN6~nGdRm<`Vx3z7ur0RPt;=k#SU&`?Eu|fvG0ymUhVf5 z29|aKR_o#?#4Q0!F4rr2(`o8X#3(Xp%4#v!%PBLZ=+Vc*=GLQ?Ui1ArZwZ&d^m{mY z!Ykege5RhDdgJIr6QGanqu0)u+v<)x_>v7Q<;TE(RtfbBK~@)zWZ#iyP1|=(qMHg) z+CHfUr6oDguH#Z}!qgkFqWYKbSxbhrCk^+e|Iw-K>>t50(pCxS2}}hj=b_u6ZB_Yv z{bmSanoHQu_RcO05LAKitIuRGu6t30X$xuQB`cnTQZPwGDJ?5~aXA9m?K={(<@-8#dvcs?xTxkwW;7U%WANiLll>J#9N>opj?v_lazS|H=|)qR#8QJ5nOMEl@`0$-B|-VMEcd-<-6Fwckxof13D#D2L!S-$o zOj*T~FYjD&}sVh^M^-IGk4Zlg64@(j}J-|BcofPyN z9e8zu`WX%QWK7S8aekQ%OY*%rGaScxRfDFLiTDOOY%;Un7`kQJ-4WR{tfn{yJ`o07`k(HczZ6E(vzdW}HEj}qQSrIgKf31J;vEQmHnp&Z9(MC>VGM@SkJLe>GnpHtSqki*veV)dN#)CZ zu??7(#(%}plF+%wDoydsH0xYRVhuRPLlH+KAR#j%OF%8Ed+MDD-!fAUPyyiUj2k?8 zZ@G8xd|^vbfehpMVW(emDCqJ6{L3;k+D$)7&e@98-2{Eb8ao>4;)N1AJ%PMnv4>vs zxIE31&JK!2CcNE6YqB6A^ly98pCFjiVK1#TZ%(K{YHwchH3|kZvjl;XO?yh08$XA| zW0Z8nyV;ULS<<*V4caCve(<0G$i7hun1_fAt^N3A7H`}nguE_tVb~y^(LwA2zIj5> zARfV9bOGO^uGGf(ApWu>)LdurJ^>!H`ic5`RXEjJMW>wqcF|l_`7&ouhddAw_(j-*!(3UIP^+tD~CdPw_yGL5Wln&9^ zjWYcYAsOK)EZ-c?>;lF{wVhpy(NWvtM%Ro+A9@ z3#+H|Hvay}ThzBav#mma(6WKgtZoK zGkKU!2*0hKe^5DAj-iYx5V!3L!-lPF(rS@fiRX$Ev2|}cp*zoiKsj0r8NQD>2>^_n<03Fj>_Ff1X`KHo0ml)S zq${GM2I^eh7NU`~wT_;_hXmj0r=j>e7tCtJk8E9Lb6$v}*E7l+X1rnC!6(#YX7@#2{I;%c_W$3(rOg`tkDVJS4Dj=f5r08b1IXyLn?E|J~B|sxkZ0-#9oB2t0?xVfw%a*7e_Rr z@*tdFnEX#gZ!Xxvul>uRE%8$xBQh#nP5ZG!d~uy4@&UQ-KKy5=a4=emJffk*ak!v4 z9ZPkyNa>1DF$3lPm@Lgqr6C&31!GOXmUb16=<|U#H=)D# zsiHHcms65HKS&;U$o5AWI(POhlgjnvhVHiMdQdJ{4qVAxD9ws3mq`oyg30fENLg%B zi$=2pHwY#Z@H20aSS8W+>nU?NadUqBY+l;adct=sC)(ILnUs3$J2Q>ZSmaOqz*1N6 zxAMldoL<@I74+-;5)y4;IAmv8NN;Et%0HpX=u zxV@2qtDDOmQq#?0xR@){v$1;}O9_(+De_mh4(*TVR*}|ud?$?m$o)T6wyZ=}m(Pns U9rPf7PXq=doEf(IlH0@o0Ic;0-v9sr diff --git a/src/static/img/lisn-logo.svg b/src/static/img/lisn-logo.svg new file mode 100644 index 000000000..e0bfdf4aa --- /dev/null +++ b/src/static/img/lisn-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/static/img/paris-saclay-logo.png b/src/static/img/paris-saclay-logo.png deleted file mode 100644 index 093966ac5c8c00cf24a597c4d6300c014064ea64..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4504 zcmV;J5ohj+P))+qsbaZqxGc)h+?~##_i;Ih8Wo02D zAz)x&w6wIRr>FV(`Ptdo%gf7zgoM=8)WN~QQBhHCZEZn8K`bmReSLjON=h3W8#+2V zPft%05)xWkT0TBLA0HnS6cpfeCJX=o5YVJ8s%e?ph9{SSP16#!4_nH>xM_LX2x1j&o0g`XIAvx=aA;bV*5bGtX4znRYFdsC z9XhlEfAZDtvuPpPj92BB8-Fz|NNZs|LV44ov=zwy@1lho0bVGcH!Vy%K^zfXq)G#L zB@`^07N?CkbrduXLcv?p(zFs}g{)~|+KB~X;Hhb0S_->5vus+F|4+o_i@t+yY3zYm z-L>xi(oe6`vi#xFv=EEicBL-#p*dWXt?mj&9R-b}qo8qg6f};G zg2vHN&^Z44DENNMr=#`a$5yD{7pu`KznrVw^LjZfehbO(hr`S7N>Az;3fAR0kB8lK zx;qGOCKwhNhyji-`{E*XwZ=u-{zbhcu^TzFo&1JG-LP<;FdQ6xGF{Yr_qluFIk8^& zW&9p2k-56q7L%!wlWlD_-=*0|D*i!9k--0*`XB45*=C3!+^%1%R1<76I@61EZ7um) zQ(FmMooIP^Gr!iFRRP3zCsOVkAokd)J{NcE^h{h2)ED)&sVtkj5k7r-cAul~ocV6S zU_y-C9K&C}(%wS>-Ih5q6zsStGgc6D0F0&J48`_{bkx_KqXAsLZ1*fn~`YP%Fgchg4hDDveaFjN^ z<5@)b+7!r}>~AHKM}B>^G6mw176sg=a0<-fR6@7a>*e|))-BZ7b{Z$7_1hD#d?U|9L9j zc^TRyr`_q)gVW)g_nVDI+x+u7@LX1dRD2R|1!)1JcME))imjXDby3AWSN1qG@#LGo zq9C<)=)$HuDiJ6!$5RO}{mtq5wgjMzh6Lw08J(J&o1*1lG+l#{3XNci0{>ssP)}`3 zmjXa|USPKKO^`}o3~$n1e?tMS@5at{_7O@Hz<`+ZmqL3-wvMT{G$P7`iBRiSCqk*y z5j~O>x+nV9HstVIC~#cVq(bVjbydVfO!@7Lf;F1?)U0(ueV*MLI|k#P&-8<+K^Dw_(CjCV zqyXx=Fk^&nlY0wBoRgFbPtXtrk{3#A(CZyBs6l~X zN?6P!Z6blIQy>*j_p@}SB2xfO)-e(G{0X6ALG#}8{E=t&4cwOoJJ+B9I!18{26G!W z8LCTxILi>N4VeOXj#I*j)lsPE;xrDXtk^C!49JsjGs7xt2Q?_ry{PAC3AxhMr$9XD z7t}=(JrsalO16Z;Xo9Ly9H87z+K_k0P#|~rU*>u=3hqCuL4kg5ED1bNPJ_L$~CmdRE!I<;N4+{3}vBR#F&C35Bhw2o7E77wElz1LkmjZEeSc!rm z&kXnD`U}Bp$7cy=E)J;Ix2y%eaE)=|F^;NJp!JoWMNyA%*}bhf3cyo<%A%J9Nd1lO zLc(wG_A>A8%@e*})2l-PTc?rX-lmfqpaASA>y-YbZ^xDlq?G{mQ<3{vh7b2c4%Z%u zuX9wVAhfQA;Pz8Gxd{qj&SlI)U5iLd{k*SOXpGY-!4t0p-+$(|n(7n``BNphFVe|P zPymH7%1VWtPS5QFryy)|o`qvuodvj7P~e!L2kGuR#{%9Fl?Is{*!p5lu2-iZu&##j zU(^wqm+MPi9bPyCP}ygCKq%=GUBP99AGbo509(9PI&E9max{*PT$qu?fm zwPf&cP@MuOjwi;fL}1bN+J7_!(D(F_ByLp7N_u7pl_|LD4wfYFw7iCcLfaD+cY5mw z?xkzd!zq9pUQOTx`>qJ|Po8i|uhi8}-q32mJy8t`bc?O{v9h_R+n)FUY{ev=RB)&k&C^;ntA4^fMP9)EqERkJkQ z3}lBptkrVox(QHtBn8GI(?g=<5Cu8JNbGa!N?u06R9vj7Oo3>kftN_^*RM|$k4QGL zWo}<nHPW?!ghe4NUvr=XhZ9ZQM1%~tmk$zRsF!n zhc@Yv^oV z5bezij#H~pNe?Jy_NMPK`v%=(DZr7)Ue_9KD&u`JO7{56!GwV@1$3ogetp!_z^3Im ztB(TVd#yOn2fmQSqfxxYFO{clJ_ldA`Vk6H;JPo1=c{ZQ6T1GZQ#~C~7I+XHiM$l> z9U>$TTa?4Nt2?$hYUMnb0_%tKkdjF2No@akpB$$7-N*i%1{S>^qx4L;eb$TpM&2;$~akn|r4^aTi!(|`WkyN+k z^R-e6#3eI6+x1h>`{V3oVtL+#kGiI1Ual3nX6StT(KABXO4}{b&5|x&oIz)PkDKvhIv6k0$uf(f0-awK zNH(1&hAW9T{84G&(hUlqyZnYbWbB=$+gUNW8gHi_%0dFtw?1v>%jGy5uEnDRzNNxg ze5{83&89zGc{kTD;?1qfq*smR!9KP_hYlS&bm-8bLx&C>I&|pJp~Hjl>+^rbFBA7I zh9kb+r`H=!Zs1<8KE@z%oWrB+nwk+;@+mHyrju-X#|LOXRor>H5l>I-J*QWhQlIzr zuvd=Ew7a1f{P+)^UuDb>v+CXwgRFL1M(i7U_lNcR1RqD zOvY=EbBSvVy!~P<1VF`tR~)b=erObzqE={wVRq`Cn5B$aihJ@^eXxVnW{!7OFK_S| zJw^eE0Z|m#R^T2-)(<(8&(40ljW23ZSU7rRB{F*_$|-;_Ae6Ll-!u%cUkm&!J4e}2 zrI_Rb1t<>4rGRa646Ttdq1I}jGEN+~e$Jb0;0ud70brWDqfZm$XT z#8HwYPeUl!gFi=`CU;Rpb^7gO2Mi4XAOQJguM;Fo9BhJeJ zb)aPwsO!oY!?v6balW<1J*i9o29$yx)uXmZpH#zl^yX0D+$vGvInc(1dTC&{kg`Z! z(R-%L@P%TOQJ@%jcB)IyScRrj2*m?chf|IT{hMe-7QKvg9w^f4GC+Y2ej1B=YZ+AL zrB_!c z$bN;9_u~=xwYn`N%n`7J7pqrVyY%E#AQxc@E=Kc_<^lm9FHwHDV>J*8w6PbHi$$lb zG)khD2n8qYQSce1JsgZq;t>0Dz<5h<24vJ6elssp!=$xKmH7T|+lDFF8cir_X{TwO z9=`LzYD0fBb9#k2kO7LUj&Un`K2>gFN=6c#0%6mHX3QZSMFv{sIpL$EQ$XgK+L~cH zE9Fb@eI}rUf>&EYH6T+{*hCdq+*EuAgxRq%qz6c=Gpga9Oo6RAjDp^UMp?qZf?`=w z{XNl!fW};9k*cbw^kVE%)e8Jwb&vveQT2LCQfl|Ww5{}T&C1{u7)7W7V76Xeu~sU< zcB#MsDjlcbYVfbIw^S#}6kM^}^2|MGAjztbieCJ1^EwDpP+Xhrisudy%~Tuz&_Qaz zD6l4-aMYT39w;Eg)kdP=hW7+|9<2)05wZGv8gaiF7s~oC)L*afBSe9;^MmG4-2dyb zxW_1%qd`d8EP{&XUZVBGM?ueYtGfT#qa`CD^IJ+EzII=WsZ(Qv_t2IQ1=80~s2f62 zQ1O(Nd!SyOZ8n>15EC;z7lguNUGabUd^-JJF?s|FkZpMoD5F3&&=*zflFMXg0-7J9 zWK?Yf6o_9dqIT(HDG*0_`Kvfqlf_?E0rjR)e78$M9OdS>GA09_x+%7Sip*enzHx~n z)OqUv=mg*ljh_JuKkZZlH~bmbk74j&&2UE{n1n>K7_$>H>I&_}szy(xq^3=t2vZzudj{px`xEm9}M2 zb9LWjM=vi0N(s~}s|tYX;7?PKJJO>>STZX9*tweSiolb{;)sZUqAU)I_$S7u!7K$; z=|p6_Eb#ta9I+&9TrN5y#iBEvxH#q0{qO@Qm_FUWl2^CX*i}@KH^zVL>cw|lT!d<~ zEB=us$8#QMcvx$xsT4aKNhcWEvdsTHO_I;SCZX`nQf=Ux4S6r$B2uO qIxn8!DmN)I*h-{w{o4hz8~y_XB+POn+_ka*0000 + + + +logo +Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/static/stylus/base_template.styl b/src/static/stylus/base_template.styl index 381dd3895..53e3e7d8b 100644 --- a/src/static/stylus/base_template.styl +++ b/src/static/stylus/base_template.styl @@ -37,10 +37,12 @@ html .organization-logos position absolute bottom 5px - right 5px + right 0 + .organization-logo height 30px + margin-right 5px #home-logo margin-left: 8px; diff --git a/src/templates/pages/home.html b/src/templates/pages/home.html index 6228a520f..da1b7bbab 100644 --- a/src/templates/pages/home.html +++ b/src/templates/pages/home.html @@ -10,11 +10,19 @@
- - - + + + + + + + + + + + +
-
{% endblock %} From 174c6ff3fee6b697f90462b658bde93820e587eb Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 10 Nov 2023 20:20:44 +0500 Subject: [PATCH 049/136] white cnrs logo added --- src/static/img/cnrs-logo.svg | 32 +++++++++----------------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/src/static/img/cnrs-logo.svg b/src/static/img/cnrs-logo.svg index 97af5c462..692f4aca3 100644 --- a/src/static/img/cnrs-logo.svg +++ b/src/static/img/cnrs-logo.svg @@ -1,29 +1,15 @@ - - - + - + - - - - + + + + - + \ No newline at end of file From 10a819a5ed83ab0c9ebedf17154b1d02795d1339 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 12 Nov 2023 15:00:06 +0500 Subject: [PATCH 050/136] in server status, show parent's submission queue for child submissions --- src/apps/pages/views.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/apps/pages/views.py b/src/apps/pages/views.py index a555e80fa..c3ade0ae4 100644 --- a/src/apps/pages/views.py +++ b/src/apps/pages/views.py @@ -90,8 +90,14 @@ def get_context_data(self, *args, **kwargs): for submission in context['submissions']: # Get filesize from each submissions's data submission.file_size = self.format_file_size(submission.data.file_size) + # Get queue from each submission - queue_name = "*" if submission.queue is None else submission.queue.name + queue_name = "" + # if submission has parent get queue from parent otherwise from the submission iteset + if submission.parent: + queue_name = "*" if submission.parent.queue is None else submission.parent.queue.name + else: + queue_name = "*" if submission.queue is None else submission.queue.name submission.competition_queue = queue_name return context From 45b639892d927f0806748fa7106f76a2d9b12b72 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 13 Nov 2023 16:04:37 +0500 Subject: [PATCH 051/136] default value of has_max_submissions changed to True in Phase table --- .../migrations/0040_auto_20231113_1103.py | 18 ++++++++++++++++++ src/apps/competitions/models.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/apps/competitions/migrations/0040_auto_20231113_1103.py diff --git a/src/apps/competitions/migrations/0040_auto_20231113_1103.py b/src/apps/competitions/migrations/0040_auto_20231113_1103.py new file mode 100644 index 000000000..2dc06eb14 --- /dev/null +++ b/src/apps/competitions/migrations/0040_auto_20231113_1103.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.17 on 2023-11-13 11:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0039_merge_20230906_1305'), + ] + + operations = [ + migrations.AlterField( + model_name='phase', + name='has_max_submissions', + field=models.BooleanField(default=True), + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index f139ce29b..5bef5825b 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -280,7 +280,7 @@ class Phase(ChaHubSaveMixin, models.Model): has_been_migrated = models.BooleanField(default=False) hide_output = models.BooleanField(default=False) - has_max_submissions = models.BooleanField(default=False) + has_max_submissions = models.BooleanField(default=True) max_submissions_per_day = models.PositiveIntegerField(default=5, null=True, blank=True) max_submissions_per_person = models.PositiveIntegerField(default=100, null=True, blank=True) From 6a84ff3f8e31014ca1e498d9e3fad607f0603048 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 13 Nov 2023 17:47:58 +0500 Subject: [PATCH 052/136] super users can access private competitions without secret key and can edit them --- src/apps/api/views/competitions.py | 50 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index fa34b4d55..f5193b8b6 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -112,32 +112,34 @@ def get_queryset(self): # not called from search bar # not called with a valid secret key if (not mine) and (not participating_in) and (not secret_key) and (not search_query): - - # Return the following --- - # All competitions which belongs to you (private or public) - # And competitions where you are admin - # And public competitions - # And competitions where you are approved participant - # this filters out all private compettions from other users - base_qs = qs.filter( - (Q(created_by=self.request.user)) | - (Q(collaborators__in=[self.request.user])) | - (Q(published=True) & ~Q(created_by=self.request.user)) | - (Q(participants__user=self.request.user) & Q(participants__status="approved")) - ) - - # Additional condition of action - # allow private competition when action is register and has valid secret key - if self.request.method == 'POST' and self.action == 'register': - # get secret_key from request data - register_secret_key = self.request.data.get('secret_key', None) - # use secret key if available - if register_secret_key: - qs = base_qs | qs.filter(Q(secret_key=register_secret_key)) + # If authenticated user is not super user + if not self.request.user.is_superuser: + # Return the following --- + # All competitions which belongs to you (private or public) + # And competitions where you are admin + # And public competitions + # And competitions where you are approved participant + # this filters out all private compettions from other users + base_qs = qs.filter( + (Q(created_by=self.request.user)) | + (Q(collaborators__in=[self.request.user])) | + (Q(published=True) & ~Q(created_by=self.request.user)) | + (Q(participants__user=self.request.user) & Q(participants__status="approved")) + ) + + # Additional condition of action + # allow private competition when action is register and has valid secret key + if self.request.method == 'POST' and self.action == 'register': + # get secret_key from request data + register_secret_key = self.request.data.get('secret_key', None) + # use secret key if available + if register_secret_key: + qs = base_qs | qs.filter(Q(secret_key=register_secret_key)) + else: + qs = base_qs else: qs = base_qs - else: - qs = base_qs + # select distinct competitions qs = qs.distinct() From 5985291305017c0ecf5706adbf94b2f76f8f5d67 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 20 Nov 2023 20:06:22 +0500 Subject: [PATCH 053/136] Competition Whitelist Emails feature (#1223) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * competition whitelist feature added * updated whitelist * Update whitelist note --------- Co-authored-by: Adrien Pavão --- src/apps/api/serializers/competitions.py | 37 ++++++++- src/apps/api/views/competitions.py | 21 ++++- .../0040_competitionwhitelistemail.py | 22 +++++ .../migrations/0041_auto_20231112_1446.py | 19 +++++ src/apps/competitions/models.py | 12 +++ src/static/js/ours/utils.js | 16 +++- .../competitions/editor/_participation.tag | 83 ++++++++++++++++++- 7 files changed, 200 insertions(+), 10 deletions(-) create mode 100644 src/apps/competitions/migrations/0040_competitionwhitelistemail.py create mode 100644 src/apps/competitions/migrations/0041_auto_20231112_1446.py diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index 2f580903b..2d1e1639c 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -9,7 +9,7 @@ from api.serializers.profiles import CollaboratorSerializer from api.serializers.submissions import SubmissionScoreSerializer from api.serializers.tasks import PhaseTaskInstanceSerializer -from competitions.models import Competition, Phase, Page, CompetitionCreationTaskStatus, CompetitionParticipant +from competitions.models import Competition, Phase, Page, CompetitionCreationTaskStatus, CompetitionParticipant, CompetitionWhiteListEmail from forums.models import Forum from leaderboards.models import Leaderboard from profiles.models import User @@ -208,6 +208,12 @@ class Meta: ) +class CompetitionWhitelistSerializer(serializers.ModelSerializer): + class Meta: + model = CompetitionWhiteListEmail + fields = ['email'] + + class CompetitionSerializer(DefaultUserCreateMixin, WritableNestedModelSerializer): created_by = serializers.CharField(source='created_by.username', read_only=True) pages = PageSerializer(many=True) @@ -217,6 +223,7 @@ class CompetitionSerializer(DefaultUserCreateMixin, WritableNestedModelSerialize # We're using a Base64 image field here so we can send JSON for create/update of this object, if we wanted # include the logo as a _file_ then we would need to use FormData _not_ JSON. logo = NamedBase64ImageField(required=True, allow_null=True) + whitelist_emails = CompetitionWhitelistSerializer(many=True, required=False) class Meta: model = Competition @@ -247,6 +254,7 @@ class Meta: 'reward', 'contact_email', 'report', + 'whitelist_emails' ) def validate_phases(self, phases): @@ -287,6 +295,26 @@ def create(self, validated_data): return instance + def update(self, instance, validated_data): + + # Get the updated whitelist emails from the validated data + updated_whitelist_emails = validated_data.get('whitelist_emails', []) + + # Delete all existing emails + instance.whitelist_emails.all().delete() + + # Save the updated whitelist emails to the instance + for whitelist_email in updated_whitelist_emails: + CompetitionWhiteListEmail.objects.create(competition=instance, email=whitelist_email["email"]) + + # Remove the 'whitelist_emails' key from validated_data to prevent it from being processed again + validated_data.pop('whitelist_emails', None) + + # Continue with the regular update process + super(CompetitionSerializer, self).update(instance, validated_data) + + return instance + class CompetitionUpdateSerializer(CompetitionSerializer): phases = PhaseUpdateSerializer(many=True) @@ -307,6 +335,7 @@ class CompetitionDetailSerializer(serializers.ModelSerializer): participant_count = serializers.IntegerField(read_only=True) submission_count = serializers.IntegerField(read_only=True) queue = QueueSerializer(read_only=True) + whitelist_emails = serializers.SerializerMethodField() class Meta: model = Competition @@ -340,6 +369,7 @@ class Meta: 'reward', 'contact_email', 'report', + 'whitelist_emails' ) def get_leaderboards(self, instance): @@ -352,6 +382,11 @@ def get_leaderboards(self, instance): raise Exception(f'KeyError on context. Context: {self.context}') return LeaderboardSerializer(qs, many=True).data + def get_whitelist_emails(self, instance): + whitelist_emails_query = instance.whitelist_emails.all() + whitelist_emails_list = [entry.email for entry in whitelist_emails_query] + return whitelist_emails_list + class CompetitionSerializerSimple(serializers.ModelSerializer): created_by = serializers.CharField(source='created_by.username') diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index f5193b8b6..565cdee27 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -253,7 +253,7 @@ def update(self, request, *args, **kwargs): # save leaderboard individually, then pass pk to each phase if 'leaderboards' in data: leaderboard_data = data['leaderboards'][0] - if(leaderboard_data['id']): + if leaderboard_data['id']: leaderboard_instance = Leaderboard.objects.get(id=leaderboard_data['id']) leaderboard = LeaderboardSerializer(leaderboard_instance, data=data['leaderboards'][0]) else: @@ -298,8 +298,16 @@ def update(self, request, *args, **kwargs): phase['starting_kit'] = Data.objects.filter(key=phase['starting_kit']['value'])[0].id except TypeError: phase['starting_kit'] = None + + # Get whitelist emails from data + whitelist_emails = data['whitelist_emails'] + # Delete white_list emails from data because it is not in a list of dict format, it is just list of emails + data.pop('whitelist_emails', None) + # Loop over whitelist emails and add them back to whitelist emails in dict format + for email in whitelist_emails: + data.setdefault('whitelist_emails', []).append({'email': email}) + serializer = self.get_serializer(instance, data=data, partial=partial) - type(serializer) serializer.is_valid(raise_exception=True) self.perform_update(serializer) @@ -342,8 +350,13 @@ def register(self, request, pk): participant.status = 'approved' send_participation_accepted_emails(participant) else: - participant.status = 'pending' - send_participation_requested_emails(participant) + # check if user is in whitelist emails then approve directly + if user.email in list(competition.whitelist_emails.values_list('email', flat=True)): + participant.status = 'approved' + send_participation_accepted_emails(participant) + else: + participant.status = 'pending' + send_participation_requested_emails(participant) participant.save() return Response({'participant_status': participant.status}, status=status.HTTP_201_CREATED) diff --git a/src/apps/competitions/migrations/0040_competitionwhitelistemail.py b/src/apps/competitions/migrations/0040_competitionwhitelistemail.py new file mode 100644 index 000000000..5be18d55a --- /dev/null +++ b/src/apps/competitions/migrations/0040_competitionwhitelistemail.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2.17 on 2023-11-12 14:44 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0039_merge_20230906_1305'), + ] + + operations = [ + migrations.CreateModel( + name='CompetitionWhiteListEmail', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.EmailField(max_length=254)), + ('competition', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='competitions.Competition')), + ], + ), + ] diff --git a/src/apps/competitions/migrations/0041_auto_20231112_1446.py b/src/apps/competitions/migrations/0041_auto_20231112_1446.py new file mode 100644 index 000000000..d65214d5f --- /dev/null +++ b/src/apps/competitions/migrations/0041_auto_20231112_1446.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.17 on 2023-11-12 14:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0040_competitionwhitelistemail'), + ] + + operations = [ + migrations.AlterField( + model_name='competitionwhitelistemail', + name='competition', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='whitelist_emails', to='competitions.Competition'), + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 5bef5825b..65ff5c750 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -678,3 +678,15 @@ class CompetitionDump(models.Model): def __str__(self): return f"Comp dump created by {self.dataset.created_by} - {self.status}" + + +# Competition White List Email Model class +# related to Competition Model +# Each Competition can have multiple white list emails +# These are used to auto approve if competition white list has this email +class CompetitionWhiteListEmail(models.Model): + competition = models.ForeignKey(Competition, on_delete=models.CASCADE, related_name='whitelist_emails') + email = models.EmailField() + + def __str__(self): + return f"{self.email} - Competition: {self.competition.title}" diff --git a/src/static/js/ours/utils.js b/src/static/js/ours/utils.js index c590ff845..c129b1d8c 100644 --- a/src/static/js/ours/utils.js +++ b/src/static/js/ours/utils.js @@ -151,13 +151,23 @@ const easyMDE_rendering_config = { } } -function create_easyMDE(element) { +function create_easyMDE(element, showToolBar = true, showStatusBar = true, editorHeight = '300px') { + + var toolbarIcons = [] + if(showToolBar){ + toolbarIcons = ["bold", "italic", "heading", "|", "quote", "unordered-list", "ordered-list", "|", "link", "image", "|", "preview", "guide"] + } + let statusItems = ["lines", "words", "cursor"] + + var markdown_editor = new EasyMDE({ element: element, autoRefresh: true, forceSync: true, - hideIcons: ["side-by-side", "fullscreen"], - renderingConfig: easyMDE_rendering_config + toolbar: toolbarIcons, + renderingConfig: easyMDE_rendering_config, + status: showStatusBar ? statusItems : showStatusBar, + minHeight: editorHeight || '300px' // Adjust the height, default is 300 }) element.EASY_MDE = markdown_editor return markdown_editor diff --git a/src/static/riot/competitions/editor/_participation.tag b/src/static/riot/competitions/editor/_participation.tag index d35d3d8cc..2a5463527 100644 --- a/src/static/riot/competitions/editor/_participation.tag +++ b/src/static/riot/competitions/editor/_participation.tag @@ -4,7 +4,7 @@ -
+
+ + + +
+ +

A list of emails (one per line) of users who do not require competition organizer's approval to enter this competition.

+
+ Note:
+ Only valid emails are allowed
+ Empty lines are not allowed +
+ +
+
From ab64b0881404890723f258629bb0e6538d020f87 Mon Sep 17 00:00:00 2001 From: OhMaley Date: Wed, 29 Nov 2023 16:53:54 -0500 Subject: [PATCH 059/136] add visual feedback on bulndle upload quota error --- src/static/riot/competitions/upload.tag | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/static/riot/competitions/upload.tag b/src/static/riot/competitions/upload.tag index 061eb27ea..e1e564b48 100644 --- a/src/static/riot/competitions/upload.tag +++ b/src/static/riot/competitions/upload.tag @@ -98,6 +98,8 @@ return } + self.clear_form() + // Call the progress bar wrapper and do the upload -- we want to check and display errors // first before doing the actual upload self.prepare_upload(self.upload)() @@ -135,7 +137,6 @@ }) .always(function () { setTimeout(self.hide_progress_bar, 500) - self.clear_form() }) } From d70f66e82b28d34c5774d84d56d806b09a3192f9 Mon Sep 17 00:00:00 2001 From: dtuantran Date: Tue, 12 Dec 2023 17:48:11 +0100 Subject: [PATCH 060/136] Add RabbitMQ http proxy environment variables example --- .env_sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env_sample b/.env_sample index 258758d54..b39a83de8 100644 --- a/.env_sample +++ b/.env_sample @@ -24,6 +24,9 @@ RABBITMQ_DEFAULT_PASS=rabbit-password-you-should-change RABBITMQ_MANAGEMENT_PORT=15672 RABBITMQ_PORT=5672 WORKER_CONNECTION_TIMEOUT=100000000 # milliseconds +#RABBITMQ_HTTP_PROXY=http://proxy-example:3128 +#RABBITMQ_HTTPS_PROXY=http://proxy-example:3128 +#RABBITMQ_NO_PROXY=localhost,172.0.0.0/8 FLOWER_PUBLIC_PORT=5555 From 3a188f6ecfbbe4320bba638dc9cd6f4a59e0b02c Mon Sep 17 00:00:00 2001 From: dtuantran Date: Tue, 12 Dec 2023 17:50:26 +0100 Subject: [PATCH 061/136] Add http proxy environment variables for rabbitmq --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d45c6d013..8d0a6359c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -135,6 +135,10 @@ services: # containers being destroyed..! hostname: rabbit env_file: .env + environment: + - http_proxy=${RABBITMQ_HTTP_PROXY:} + - https_proxy=${RABBITMQ_HTTPS_PROXY:} + - no_proxy=${RABBITMQ_NO_PROXY:} ports: - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 - ${RABBITMQ_PORT}:5672 From cfeea103e77af8af6c8c1148682a66137ac3dd2b Mon Sep 17 00:00:00 2001 From: dtuantran Date: Tue, 12 Dec 2023 18:05:51 +0100 Subject: [PATCH 062/136] No need ":" for default value --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8d0a6359c..fa5cadb58 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,9 +136,9 @@ services: hostname: rabbit env_file: .env environment: - - http_proxy=${RABBITMQ_HTTP_PROXY:} - - https_proxy=${RABBITMQ_HTTPS_PROXY:} - - no_proxy=${RABBITMQ_NO_PROXY:} + - http_proxy=${RABBITMQ_HTTP_PROXY} + - https_proxy=${RABBITMQ_HTTPS_PROXY} + - no_proxy=${RABBITMQ_NO_PROXY} ports: - ${RABBITMQ_MANAGEMENT_PORT:-15672}:15672 - ${RABBITMQ_PORT}:5672 From 5bfd601b6cde6a89ae3816854c8faf1a04632875 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 14 Dec 2023 20:55:47 +0500 Subject: [PATCH 063/136] flake resolved --- src/apps/analytics/models.py | 1 - src/apps/analytics/tasks.py | 69 ++++++++++++--------------------- src/apps/api/views/analytics.py | 11 +++--- src/apps/competitions/models.py | 10 ++--- src/apps/datasets/models.py | 2 +- src/settings/base.py | 4 +- 6 files changed, 39 insertions(+), 58 deletions(-) diff --git a/src/apps/analytics/models.py b/src/apps/analytics/models.py index 9c1ade208..f0afe09f3 100644 --- a/src/apps/analytics/models.py +++ b/src/apps/analytics/models.py @@ -1,6 +1,5 @@ from django.db import models from django.conf import settings -from competitions.models import Competition class StorageUsageHistory(models.Model): diff --git a/src/apps/analytics/tasks.py b/src/apps/analytics/tasks.py index 7fd2b5bed..85d13d04e 100644 --- a/src/apps/analytics/tasks.py +++ b/src/apps/analytics/tasks.py @@ -24,9 +24,7 @@ AdminStorageDataPoint, ) from competitions.models import Competition -from datasets.models import Data from profiles.models import User -from competitions.models import Submission, SubmissionDetails from utils.data import pretty_bytes @@ -45,7 +43,7 @@ def create_storage_analytics_snapshot(): dataset.file_size = Decimal( dataset.data_file.size / 1024 ) # file_size is in KiB - except: + except Exception: dataset.file_size = Decimal(-1) finally: dataset.save() @@ -58,7 +56,7 @@ def create_storage_analytics_snapshot(): submission.prediction_result_file_size = Decimal( submission.prediction_result.size / 1024 ) # prediction_result_file_size is in KiB - except: + except Exception: submission.prediction_result_file_size = Decimal(-1) finally: submission.save() @@ -70,7 +68,7 @@ def create_storage_analytics_snapshot(): submission.scoring_result_file_size = Decimal( submission.scoring_result.size / 1024 ) # scoring_result_file_size is in KiB - except: + except Exception: submission.scoring_result_file_size = Decimal(-1) finally: submission.save() @@ -82,7 +80,7 @@ def create_storage_analytics_snapshot(): submission.detailed_result_file_size = Decimal( submission.detailed_result.size / 1024 ) # detailed_result_file_size is in KiB - except: + except Exception: submission.detailed_result_file_size = Decimal(-1) finally: submission.save() @@ -94,7 +92,7 @@ def create_storage_analytics_snapshot(): submissiondetails.file_size = Decimal( submissiondetails.data_file.size / 1024 ) # file_size is in KiB - except: + except Exception: submissiondetails.file_size = Decimal(-1) finally: submissiondetails.save() @@ -177,16 +175,14 @@ def create_storage_analytics_snapshot(): ), default=Value(0), output_field=DecimalField(), - ) - + Case( + ) + Case( When( scoring_result_file_size__gt=0, then=F("scoring_result_file_size"), ), default=Value(0), output_field=DecimalField(), - ) - + Case( + ) + Case( When( detailed_result_file_size__gt=0, then=F("detailed_result_file_size"), @@ -242,8 +238,7 @@ def create_storage_analytics_snapshot(): ).aggregate(total=Sum("size"))["total"] defaults = { "datasets_total": datasets_usage or 0, - "submissions_total": (submissions_usage or 0) - + (submissiondetails_usage or 0), + "submissions_total": (submissions_usage or 0) + (submissiondetails_usage or 0), } lookup_params = {"user_id": user.id, "at_date": date} UserStorageDataPoint.objects.update_or_create( @@ -315,9 +310,7 @@ def create_storage_analytics_snapshot(): # Datasets for dataset in Data.objects.all().order_by("id"): if ( - not dataset.data_file - or not dataset.data_file.name - or not BundleStorage.exists(dataset.data_file.name) + not dataset.data_file or not dataset.data_file.name or not BundleStorage.exists(dataset.data_file.name) ): inconsistencies["database"].append( {"model": "dataset", "field": "data_file", "id": dataset.id} @@ -327,9 +320,7 @@ def create_storage_analytics_snapshot(): # Submissions for submission in Submission.objects.all().order_by("id"): if ( - not submission.prediction_result - or not submission.prediction_result.name - or not BundleStorage.exists(submission.prediction_result.name) + not submission.prediction_result or not submission.prediction_result.name or not BundleStorage.exists(submission.prediction_result.name) ): inconsistencies["database"].append( { @@ -340,18 +331,14 @@ def create_storage_analytics_snapshot(): ) nb_missing_files += 1 if ( - not submission.scoring_result - or not submission.scoring_result.name - or not BundleStorage.exists(submission.scoring_result.name) + not submission.scoring_result or not submission.scoring_result.name or not BundleStorage.exists(submission.scoring_result.name) ): inconsistencies["database"].append( {"model": "submission", "field": "scoring_result", "id": submission.id} ) nb_missing_files += 1 if ( - submission.detailed_result - and submission.detailed_result.name - and not BundleStorage.exists(submission.detailed_result.name) + submission.detailed_result and submission.detailed_result.name and not BundleStorage.exists(submission.detailed_result.name) ): inconsistencies["database"].append( {"model": "submission", "field": "detailed_result", "id": submission.id} @@ -361,9 +348,7 @@ def create_storage_analytics_snapshot(): # Submission details for submissiondetails in SubmissionDetails.objects.all().order_by("id"): if ( - not submissiondetails.data_file - or not submissiondetails.data_file.name - or not BundleStorage.exists(submissiondetails.data_file.name) + not submissiondetails.data_file or not submissiondetails.data_file.name or not BundleStorage.exists(submissiondetails.data_file.name) ): inconsistencies["database"].append( { @@ -490,10 +475,10 @@ def create_storage_analytics_snapshot(): # Log the results log_file = ( - "/app/logs/" - + "db_storage_inconsistency_" - + current_datetime.strftime("%Y%m%d-%H%M%S") - + ".log" + "/app/logs/" + + "db_storage_inconsistency_" + + current_datetime.strftime("%Y%m%d-%H%M%S") + + ".log" ) with open(log_file, "w") as file: file.write("Database <---> Storage Inconsistency\n\n") @@ -520,27 +505,23 @@ def create_storage_analytics_snapshot(): competitions_usage = ( competitions_datasets.filter(day__lt=date).aggregate(total=Sum("size"))[ "total" - ] - or 0 + ] or 0 ) users_usage = ( ( users_datasets.filter(day__lt=date).aggregate(total=Sum("size"))[ "total" - ] - or 0 - ) - + ( + ] or 0 + ) + + ( users_submissions.filter(day__lt=date).aggregate(total=Sum("size"))[ "total" - ] - or 0 - ) - + ( + ] or 0 + ) + + ( users_submissions_details.filter(day__lt=date).aggregate( total=Sum("size") - )["total"] - or 0 + )["total"] or 0 ) ) admin_data_point = AdminStorageDataPoint.objects.filter(at_date=date).first() diff --git a/src/apps/api/views/analytics.py b/src/apps/api/views/analytics.py index 47282c97b..1cd93a4bd 100644 --- a/src/apps/api/views/analytics.py +++ b/src/apps/api/views/analytics.py @@ -163,6 +163,7 @@ def get(self, request): 'time_unit': time_unit, }) + @api_view(["GET"]) def storage_usage_history(request): """ @@ -201,7 +202,7 @@ def competitions_usage(request): """ if not request.user.is_superuser: raise PermissionDenied(detail="Admin only") - + competitions_usage = {} last_competition_storage_snapshot = CompetitionStorageDataPoint.objects.order_by("at_date").last() if last_competition_storage_snapshot: @@ -229,7 +230,7 @@ def competitions_usage(request): 'created_when': su['competition__created_when'], 'datasets': su['datasets_total'], } - + return Response(competitions_usage, status=status.HTTP_200_OK) @@ -240,7 +241,7 @@ def users_usage(request): """ if not request.user.is_superuser: raise PermissionDenied(detail="Admin only") - + users_usage = {} last_user_storage_snapshot = UserStorageDataPoint.objects.order_by("at_date").last() if last_user_storage_snapshot: @@ -268,5 +269,5 @@ def users_usage(request): 'datasets': su['datasets_total'], 'submissions': su['submissions_total'], } - - return Response(users_usage, status=status.HTTP_200_OK) \ No newline at end of file + + return Response(users_usage, status=status.HTTP_200_OK) diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 52d92ed5c..4b87ec35a 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -403,7 +403,7 @@ class SubmissionDetails(models.Model): ] name = models.CharField(max_length=50) data_file = models.FileField(upload_to=PathWrapper('submission_details'), storage=BundleStorage) - file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB + file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB submission = models.ForeignKey('Submission', on_delete=models.CASCADE, related_name='details') is_scoring = models.BooleanField(default=False) @@ -459,10 +459,10 @@ class Submission(ChaHubSaveMixin, models.Model): storage=BundleStorage) detailed_result = models.FileField(upload_to=PathWrapper('detailed_result'), null=True, blank=True, storage=BundleStorage) - - prediction_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB - scoring_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB - detailed_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB + + prediction_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB + scoring_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB + detailed_result_file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB secret = models.UUIDField(default=uuid.uuid4) celery_task_id = models.UUIDField(null=True, blank=True) diff --git a/src/apps/datasets/models.py b/src/apps/datasets/models.py index 3ca6eb53d..729e025a1 100644 --- a/src/apps/datasets/models.py +++ b/src/apps/datasets/models.py @@ -52,7 +52,7 @@ class Data(ChaHubSaveMixin, models.Model): key = models.UUIDField(default=uuid.uuid4, blank=True, unique=True) is_public = models.BooleanField(default=False) upload_completed_successfully = models.BooleanField(default=False) - file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB + file_size = models.DecimalField(max_digits=10, decimal_places=2, null=True, blank=True) # in KiB # This is true if the Data model was created as part of unpacking a competition. Competition bundles themselves # are NOT marked True, since they are not created by unpacking! diff --git a/src/settings/base.py b/src/settings/base.py index 7a76f9221..d7095773d 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -226,11 +226,11 @@ }, 'create_storage_analytics_snapshot': { 'task': 'analytics.tasks.create_storage_analytics_snapshot', - 'schedule': crontab(hour='2', minute='0', day_of_week='sun') # Every Sunday at 02:00 UTC time + 'schedule': crontab(hour='2', minute='0', day_of_week='sun') # Every Sunday at 02:00 UTC time }, 'reset_computed_storage_analytics': { 'task': 'analytics.tasks.reset_computed_storage_analytics', - 'schedule': crontab(hour='2', minute='0', day_of_month='1', month_of_year="*/3") # Every 3 month at 02:00 UTC on the 1st + 'schedule': crontab(hour='2', minute='0', day_of_month='1', month_of_year="*/3") # Every 3 month at 02:00 UTC on the 1st }, } CELERY_TIMEZONE = 'UTC' From aa3edcbca5bec6ab71d68d7bbf6a4cc1589ca86b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 14 Dec 2023 21:01:49 +0500 Subject: [PATCH 064/136] flake fixed --- src/apps/analytics/tasks.py | 4 ++-- src/apps/profiles/models.py | 10 +++++----- src/settings/base.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/apps/analytics/tasks.py b/src/apps/analytics/tasks.py index 85d13d04e..776baf05f 100644 --- a/src/apps/analytics/tasks.py +++ b/src/apps/analytics/tasks.py @@ -49,8 +49,8 @@ def create_storage_analytics_snapshot(): dataset.save() for submission in Submission.objects.filter( - Q(prediction_result_file_size__isnull=True) - | Q(prediction_result_file_size__lt=0) + Q(prediction_result_file_size__isnull=True) | + Q(prediction_result_file_size__lt=0) ): try: submission.prediction_result_file_size = Decimal( diff --git a/src/apps/profiles/models.py b/src/apps/profiles/models.py index ddb0005dc..518230f92 100644 --- a/src/apps/profiles/models.py +++ b/src/apps/profiles/models.py @@ -133,7 +133,7 @@ def get_chahub_data(self): def get_chahub_is_valid(self): # By default, always push return True - + def get_used_storage_space(self): from datasets.models import Data from competitions.models import Submission, SubmissionDetails @@ -157,16 +157,16 @@ def get_used_storage_space(self): ), default=Value(0), output_field=DecimalField(), - ) - + Case( + ) + + Case( When( scoring_result_file_size__gt=0, then=F("scoring_result_file_size"), ), default=Value(0), output_field=DecimalField(), - ) - + Case( + ) + + Case( When( detailed_result_file_size__gt=0, then=F("detailed_result_file_size"), diff --git a/src/settings/base.py b/src/settings/base.py index d7095773d..d5047db82 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -407,7 +407,7 @@ GS_BUCKET_NAME = GS_PUBLIC_BUCKET_NAME # Default bucket set to public bucket # Quota -DEFAULT_USER_QUOTA = 15 * 1024 * 1024 * 1024 # 15GB +DEFAULT_USER_QUOTA = 15 * 1024 * 1024 * 1024 # 15GB # ============================================================================= # Debug From ed0bd714c44a25c8b8df0a61798b4fe2de6f10da Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 14 Dec 2023 21:51:52 +0500 Subject: [PATCH 065/136] solve migration issue --- .../migrations/0043_merge_20231213_0948.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/apps/competitions/migrations/0043_merge_20231213_0948.py diff --git a/src/apps/competitions/migrations/0043_merge_20231213_0948.py b/src/apps/competitions/migrations/0043_merge_20231213_0948.py new file mode 100644 index 000000000..c74629a95 --- /dev/null +++ b/src/apps/competitions/migrations/0043_merge_20231213_0948.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2.17 on 2023-12-13 09:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0042_merge_20231120_1551'), + ('competitions', '0035_auto_20230914_1319'), + ] + + operations = [ + ] From 2b0391528c48420e84b5ba0f67d969f4a4b4ee07 Mon Sep 17 00:00:00 2001 From: OhMaley Date: Mon, 18 Dec 2023 15:38:41 -0500 Subject: [PATCH 066/136] add analytics snapshot date in API response + display it --- src/apps/api/views/analytics.py | 21 +++++++++-- .../riot/analytics/_competitions_usage.tag | 37 ++++++++++++------- src/static/riot/analytics/_usage_history.tag | 22 ++++++++--- src/static/riot/analytics/_users_usage.tag | 37 ++++++++++++------- src/static/riot/analytics/analytics.tag | 8 ++++ 5 files changed, 91 insertions(+), 34 deletions(-) diff --git a/src/apps/api/views/analytics.py b/src/apps/api/views/analytics.py index 47282c97b..238002a9d 100644 --- a/src/apps/api/views/analytics.py +++ b/src/apps/api/views/analytics.py @@ -190,8 +190,13 @@ def storage_usage_history(request): 'admin_usage': su['admin_usage'], 'orphaned_file_usage': su['orphaned_file_usage'] } + + response = { + "last_storage_calculation_date": last_storage_usage_history_snapshot.at_date.isoformat() if last_storage_usage_history_snapshot else None, + "storage_usage_history": storage_usage_history + } - return Response(storage_usage_history, status=status.HTTP_200_OK) + return Response(response, status=status.HTTP_200_OK) @api_view(["GET"]) @@ -230,7 +235,12 @@ def competitions_usage(request): 'datasets': su['datasets_total'], } - return Response(competitions_usage, status=status.HTTP_200_OK) + response = { + "last_storage_calculation_date": last_competition_storage_snapshot.at_date.isoformat() if last_competition_storage_snapshot else None, + "competitions_usage": competitions_usage + } + + return Response(response, status=status.HTTP_200_OK) @api_view(["GET"]) @@ -269,4 +279,9 @@ def users_usage(request): 'submissions': su['submissions_total'], } - return Response(users_usage, status=status.HTTP_200_OK) \ No newline at end of file + response = { + "last_storage_calculation_date": last_user_storage_snapshot.at_date.isoformat() if last_user_storage_snapshot else None, + "users_usage": users_usage + } + + return Response(response, status=status.HTTP_200_OK) \ No newline at end of file diff --git a/src/static/riot/analytics/_competitions_usage.tag b/src/static/riot/analytics/_competitions_usage.tag index b9568313e..353fb0684 100644 --- a/src/static/riot/analytics/_competitions_usage.tag +++ b/src/static/riot/analytics/_competitions_usage.tag @@ -1,15 +1,18 @@ - - - +
+ + + +

{lastSnapshotDate ? "Last snaphost date: " + pretty_date(lastSnapshotDate) : "No snapshot has been taken yet"}

+
@@ -55,6 +58,7 @@ let datetime = luxon.DateTime; + self.lastSnapshotDate = null; self.competitionsUsageData = null; self.competitionsDropdownOptions = []; self.tableSelectedDate = null; @@ -221,9 +225,11 @@ }; CODALAB.api.get_competitions_usage(parameters) .done(function(data) { - self.competitionsUsageData = data; + self.competitionsUsageData = data["competitions_usage"]; + self.lastSnapshotDate = data["last_storage_calculation_date"]; + self.update({lastSnapshotDate: data["last_storage_calculation_date"]}); self.updateCompetitionsSelectionDropdown(); - self.updateCompetitionTableCalendar(data); + self.updateCompetitionTableCalendar(data["competitions_usage"]); self.updateCompetitionsChart(); self.updateCompetitionsPieChart(); self.updateCompetitionsTable(); @@ -559,5 +565,10 @@ .chart-container { min-height: 450px; } + + .flex-row { + display: flex; + flex-direction: row; + } \ No newline at end of file diff --git a/src/static/riot/analytics/_usage_history.tag b/src/static/riot/analytics/_usage_history.tag index 50535d8a6..6a75a5e53 100644 --- a/src/static/riot/analytics/_usage_history.tag +++ b/src/static/riot/analytics/_usage_history.tag @@ -1,7 +1,11 @@ - +
+ + +

{lastSnapshotDate ? "Last snaphost date: " + pretty_date(lastSnapshotDate) : "No snapshot has been taken yet"}

+
@@ -17,6 +21,7 @@ }; self.storageUsageHistoryData = null; self.storageUsageChart = null; + self.lastSnapshotDate = null; self.one("mount", function () { self.state.startDate = opts.start_date; @@ -121,8 +126,10 @@ }; CODALAB.api.get_storage_usage_history(parameters) .done(function(data) { - self.storageUsageHistoryData = data; - self.update_storage_usage_history_chart(data); + self.storageUsageHistoryData = data["storage_usage_history"]; + self.lastSnapshotDate = data["last_storage_calculation_date"]; + self.update({lastSnapshotDate: data["last_storage_calculation_date"]}); + self.update_storage_usage_history_chart(data["storage_usage_history"]); }) .fail(function(error) { toastr.error("Could not load storage analytics data"); @@ -177,5 +184,10 @@ .chart-container { min-height: 450px; } + + .flex-row { + display: flex; + flex-direction: row; + } \ No newline at end of file diff --git a/src/static/riot/analytics/_users_usage.tag b/src/static/riot/analytics/_users_usage.tag index 5aff49e28..b83a8a52c 100644 --- a/src/static/riot/analytics/_users_usage.tag +++ b/src/static/riot/analytics/_users_usage.tag @@ -1,15 +1,18 @@ - - - +
+ + + +

{lastSnapshotDate ? "Last snaphost date: " + pretty_date(lastSnapshotDate) : "No snapshot has been taken yet"}

+
@@ -62,6 +65,7 @@ let datetime = luxon.DateTime; + self.lastSnapshotDate = null; self.usersUsageData = null; self.usersDropdownOptions = []; self.usersTableSelectedDate = null; @@ -269,9 +273,11 @@ }; CODALAB.api.get_users_usage(parameters) .done(function(data) { - self.usersUsageData = data; + self.usersUsageData = data["users_usage"]; + self.lastSnapshotDate = data["last_storage_calculation_date"]; + self.update({lastSnapshotDate: data["last_storage_calculation_date"]}); self.updateUsersSelectionDropdown(); - self.updateUsersTableCalendar(data); + self.updateUsersTableCalendar(data["users_usage"]); self.updateUsersChart(); self.updateUsersPieChart(); self.updateUsersTable(); @@ -640,5 +646,10 @@ .chart-container { min-height: 450px; } + + .flex-row { + display: flex; + flex-direction: row; + } \ No newline at end of file diff --git a/src/static/riot/analytics/analytics.tag b/src/static/riot/analytics/analytics.tag index e6ecbeb26..f0a8d99b4 100644 --- a/src/static/riot/analytics/analytics.tag +++ b/src/static/riot/analytics/analytics.tag @@ -474,6 +474,14 @@ self.update(); } + self.pretty_date = function (date_string) { + if (!!date_string) { + return luxon.DateTime.fromISO(date_string).toLocaleString(luxon.DateTime.DATE_FULL) + } else { + return '' + } + } + From 92fedf4b74565c80502e41b56339ca9c2e15612e Mon Sep 17 00:00:00 2001 From: didayolo Date: Sat, 20 Jan 2024 05:48:41 +0100 Subject: [PATCH 077/136] Rename public competitions page --- src/static/riot/competitions/public-list.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index c37eb96cc..cc9a535bd 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -1,5 +1,5 @@ -

Public Competitions

+

Public Benchmarks and Competitions

From ea016e43b15cb828afeb3c8855cc6eee51d34dd6 Mon Sep 17 00:00:00 2001 From: didayolo Date: Sat, 20 Jan 2024 06:23:24 +0100 Subject: [PATCH 078/136] Clickable username in participant manager --- src/static/riot/competitions/detail/participant_manager.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/participant_manager.tag b/src/static/riot/competitions/detail/participant_manager.tag index 5485469cf..569c8f0d8 100644 --- a/src/static/riot/competitions/detail/participant_manager.tag +++ b/src/static/riot/competitions/detail/participant_manager.tag @@ -25,7 +25,7 @@
- + From 4e3a82264ba7d6e9cb2427c3d52b27176930d3f9 Mon Sep 17 00:00:00 2001 From: didayolo Date: Sat, 20 Jan 2024 22:09:42 +0100 Subject: [PATCH 079/136] Patch PR #1285 and add show more button in front page --- src/static/riot/competitions/public-list.tag | 49 ++++++++++--------- .../tile/front_page_competitions.tag | 26 +++++++++- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index cc9a535bd..2eb854a08 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -1,6 +1,6 @@

Public Benchmarks and Competitions

-
+
{ current_page } of {Math.ceil(competitions.count/competitions.page_size)} @@ -61,32 +61,35 @@ } self.update_competitions_list = function (num) { - self.current_page = num - $('#loading').show() - $('.pagination-nav').hide() - if (self.competitions_cache[self.current_page]){ - self.competitions = self.competitions_cache[self.current_page] - history.pushState("", document.title, "?page="+self.current_page) - $('.pagination-nav > button').prop('disabled', false) - self.update() + self.current_page = num; + $('#loading').show(); // Show the loading indicator + $('.pagination-nav').hide(); // Hide pagination navigation + + // Function to handle successful data retrieval + function handleSuccess(response) { + self.competitions = response; + self.competitions_cache[self.current_page.toString()] = response; + $('#loading').hide(); // Hide the loading indicator + $('.pagination-nav').show(); // Show pagination navigation + history.pushState("", document.title, "?page=" + self.current_page); + $('.pagination-nav > button').prop('disabled', false); + self.update(); + } + // Check if data is in cache + if (self.competitions_cache[self.current_page]) { + handleSuccess(self.competitions_cache[self.current_page]); } else { - return CODALAB.api.get_public_competitions({"page":self.current_page}) + // Fetch data using AJAX call + return CODALAB.api.get_public_competitions({"page": self.current_page}) .fail(function (response) { - $('#loading').hide() - $('.pagination-nav').show() - toastr.error("Could not load competition list") - }) - .done(function (response){ - self.competitions = response - self.competitions_cache[self.current_page.toString()] = response - $('#loading').hide() - $('.pagination-nav').show() - history.pushState("", document.title, "?page="+self.current_page) - $('.pagination-nav > button').prop('disabled', false) - self.update() + $('#loading').hide(); // Hide the loading indicator + $('.pagination-nav').show(); // Show pagination navigation + toastr.error("Could not load competition list"); }) + .done(handleSuccess); } - } + }; + self.get_array_length = function (arr) { if(arr === undefined){ diff --git a/src/static/riot/competitions/tile/front_page_competitions.tag b/src/static/riot/competitions/tile/front_page_competitions.tag index a24f83823..1543d13d3 100644 --- a/src/static/riot/competitions/tile/front_page_competitions.tag +++ b/src/static/riot/competitions/tile/front_page_competitions.tag @@ -2,7 +2,7 @@
- Popular Benchmarks + Popular Benchmarks
@@ -13,11 +13,12 @@
+ Show more
- Featured Benchmarks + Featured Benchmarks
@@ -28,6 +29,7 @@
+ Show more
@@ -58,6 +60,26 @@ margin: 3em 1.5em; } + .show-more { + display: block; + width: fit-content; + margin: 20px auto; + padding: 10px 20px; + background-color: #4a6778; + color: white; + text-align: center; + border-radius: 5px; + font-size: 1.1em; + text-decoration: none; + transition: background-color 0.3s, transform 0.3s; + } + + .show-more:hover { + background-color: #467799; + transform: scale(1.05); + text-decoration: none; + } + .sub-header-link { line-height: 0.25em; } From 4fc411e20da7ed6784ae647b3f957bec989485ac Mon Sep 17 00:00:00 2001 From: didayolo Date: Sun, 21 Jan 2024 02:56:07 +0100 Subject: [PATCH 080/136] Clickable username in competition header --- src/static/riot/competitions/detail/_header.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/_header.tag b/src/static/riot/competitions/detail/_header.tag index 87d4fe870..d6cda97d4 100644 --- a/src/static/riot/competitions/detail/_header.tag +++ b/src/static/riot/competitions/detail/_header.tag @@ -41,7 +41,7 @@
Organized by: - {competition.created_by} + {competition.created_by} ({competition.contact_email})
From e9c8bd3606ddcefc9d441b4d401de4a38db54dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Sun, 21 Jan 2024 03:08:48 +0100 Subject: [PATCH 081/136] Replace "date of last entry" by "date" --- src/static/riot/competitions/detail/leaderboards.tag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/static/riot/competitions/detail/leaderboards.tag b/src/static/riot/competitions/detail/leaderboards.tag index e21238b23..0617a38ea 100644 --- a/src/static/riot/competitions/detail/leaderboards.tag +++ b/src/static/riot/competitions/detail/leaderboards.tag @@ -35,7 +35,7 @@
- + @@ -58,7 +58,7 @@ - +
{username}{username} {email} {is_bot} {_.startCase(status)}# Participant EntriesDate of last entryDate {column.title}
{ submission.owner } { submission.organization.name } {submission.num_entries}{submission.last_entry_date}{submission.created_when} From 2a941ab55ae265ce7d392d74de3d9af70bc9b537 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 22 Jan 2024 11:50:30 +0500 Subject: [PATCH 082/136] leaderboard date fixed --- src/apps/api/serializers/submissions.py | 4 +++- src/apps/api/views/competitions.py | 8 +------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/apps/api/serializers/submissions.py b/src/apps/api/serializers/submissions.py index ef3cdf39b..81e8de21c 100644 --- a/src/apps/api/serializers/submissions.py +++ b/src/apps/api/serializers/submissions.py @@ -86,6 +86,7 @@ class SubmissionLeaderBoardSerializer(serializers.ModelSerializer): display_name = serializers.CharField(source='owner.display_name') slug_url = serializers.CharField(source='owner.slug_url') organization = SimpleOrganizationSerializer(allow_null=True) + created_when = serializers.DateTimeField(format="%Y-%m-%d %H:%M") class Meta: model = Submission @@ -100,7 +101,8 @@ class Meta: 'display_name', 'slug_url', 'organization', - 'detailed_result' + 'detailed_result', + 'created_when' ) extra_kwargs = { "scores": {"read_only": True}, diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 565cdee27..3dc2f0bba 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -712,12 +712,6 @@ def get_leaderboard(self, request, pk): parent__isnull=False ).count() - # get date of last submission by the owner of this submission for this phase - last_entry_date = Submission.objects.filter(owner__username=submission['owner'], phase=phase)\ - .values('created_when')\ - .order_by('-created_when')[0]['created_when']\ - .strftime('%Y-%m-%d') - submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}" # gather detailed result from submissions for each task @@ -741,7 +735,7 @@ def get_leaderboard(self, request, pk): 'slug_url': submission['slug_url'], 'organization': submission['organization'], 'num_entries': num_entries, - 'last_entry_date': last_entry_date + 'created_when': submission['created_when'] }) for score in submission['scores']: From e293f359c99d0f8be6ef693d3bd357cea041d128 Mon Sep 17 00:00:00 2001 From: Benjamin Bearce Date: Mon, 22 Jan 2024 20:56:47 -0500 Subject: [PATCH 083/136] quota message details and link to user quote --- src/static/riot/competitions/detail/submission_upload.tag | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/static/riot/competitions/detail/submission_upload.tag b/src/static/riot/competitions/detail/submission_upload.tag index 1e2d89508..406a8ee81 100644 --- a/src/static/riot/competitions/detail/submission_upload.tag +++ b/src/static/riot/competitions/detail/submission_upload.tag @@ -488,7 +488,9 @@ } } - toastr.error("Creation failed, error occurred") + toastr.error(`Creation failed, error occurred: ${response.responseJSON.data_file[0]}`) + setTimeout(()=>{toastr.warning(`Click HERE to go to the Resources tab to see your user quote.`)}, 5000) + }) .always(function () { setTimeout(self.hide_progress_bar, 500) From e9ae970b460753a928ee407fca390fe3412b0b6b Mon Sep 17 00:00:00 2001 From: didayolo Date: Tue, 23 Jan 2024 15:03:54 +0100 Subject: [PATCH 084/136] Update quota error messages --- src/apps/api/tests/test_datasets.py | 2 +- src/apps/api/views/datasets.py | 2 +- src/static/riot/competitions/detail/submission_upload.tag | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/apps/api/tests/test_datasets.py b/src/apps/api/tests/test_datasets.py index 40b4dbf5d..944981757 100644 --- a/src/apps/api/tests/test_datasets.py +++ b/src/apps/api/tests/test_datasets.py @@ -65,7 +65,7 @@ def test_dataset_api_check_quota(self): }) assert resp.status_code == 400 - assert resp.data["data_file"][0] == f"Insufficient space. Your available space is {pretty_bytes(available_space)}. The file size is {pretty_bytes(file_size)}. Please free up some space and try again." + assert resp.data["data_file"][0] == f'Insufficient space. Your available space is {pretty_bytes(available_space)}. The file size is {pretty_bytes(file_size)}. Please free up some space and try again. You can manage your files in the Resources page.' # Fake upload a small file file_size = available_space - 1024 diff --git a/src/apps/api/views/datasets.py b/src/apps/api/views/datasets.py index 1f4fffa65..2175b9dc0 100644 --- a/src/apps/api/views/datasets.py +++ b/src/apps/api/views/datasets.py @@ -86,7 +86,7 @@ def create(self, request, *args, **kwargs): if storage_used + file_size > quota: available_space = pretty_bytes(quota - storage_used) file_size = pretty_bytes(file_size) - message = f"Insufficient space. Your available space is {available_space}. The file size is {file_size}. Please free up some space and try again." + message = f'Insufficient space. Your available space is {available_space}. The file size is {file_size}. Please free up some space and try again. You can manage your files in the Resources page.' return Response({'data_file': [message]}, status=status.HTTP_400_BAD_REQUEST) # All good, let's proceed diff --git a/src/static/riot/competitions/detail/submission_upload.tag b/src/static/riot/competitions/detail/submission_upload.tag index 406a8ee81..e463e899a 100644 --- a/src/static/riot/competitions/detail/submission_upload.tag +++ b/src/static/riot/competitions/detail/submission_upload.tag @@ -489,7 +489,7 @@ } } toastr.error(`Creation failed, error occurred: ${response.responseJSON.data_file[0]}`) - setTimeout(()=>{toastr.warning(`Click HERE to go to the Resources tab to see your user quote.`)}, 5000) + setTimeout(()=>{toastr.warning(`Click HERE to go to the Resources tab to see your storage quota and manage your files.`)}, 5000) }) .always(function () { From ec29a31b14b3561c07da2f4f22b9883b36bda1f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Tue, 23 Jan 2024 17:43:53 +0100 Subject: [PATCH 085/136] Remove warning from submission_upload.tag --- src/static/riot/competitions/detail/submission_upload.tag | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/static/riot/competitions/detail/submission_upload.tag b/src/static/riot/competitions/detail/submission_upload.tag index e463e899a..75ecc8b2f 100644 --- a/src/static/riot/competitions/detail/submission_upload.tag +++ b/src/static/riot/competitions/detail/submission_upload.tag @@ -489,8 +489,6 @@ } } toastr.error(`Creation failed, error occurred: ${response.responseJSON.data_file[0]}`) - setTimeout(()=>{toastr.warning(`Click HERE to go to the Resources tab to see your storage quota and manage your files.`)}, 5000) - }) .always(function () { setTimeout(self.hide_progress_bar, 500) From 9e48021a7010f194c8f01dfbabcfb1f5232b2582 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 28 Jan 2024 16:10:36 +0500 Subject: [PATCH 086/136] Rerun submission, display error when a task is deleted and a submission using it is rerun --- src/apps/api/views/submissions.py | 9 ++++- src/apps/competitions/models.py | 37 ++++++++++++++++--- .../detail/submission_manager.tag | 6 ++- 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index b61963f9b..f9582488d 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -270,7 +270,14 @@ def re_run_submission(self, request, pk): rerun_kwargs = {} new_sub = submission.re_run(**rerun_kwargs) - return Response({'id': new_sub.id}) + if new_sub is None: + # return error + return Response({ + "error_msg": "You cannot rerun this submission because one or more tasks this submission was running are deleted, resubmit the submission or contact the competition organizer!"}, + status=status.HTTP_404_NOT_FOUND + ) + else: + return Response({'id': new_sub.id}) @action(detail=False, methods=('POST',)) def re_run_many_submissions(self, request): diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 4b87ec35a..a35141e54 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -539,24 +539,51 @@ def start(self, tasks=None): run_submission(self.pk, tasks=tasks) def re_run(self, task=None): + + # task to use in the new submission + new_submission_task = task or self.task + + # set is_specific_task_re_run + is_specific_task_re_run = bool(task) + + flag_rerun_specific_task_or_has_no_children = False + # Check if this submission needs to rerun on specific children or has no children + if not self.has_children or is_specific_task_re_run: + flag_rerun_specific_task_or_has_no_children = True + + # Check if task exists in case of specific task rerun or no children + if flag_rerun_specific_task_or_has_no_children and new_submission_task is None: + logger.error(f"Cannot rerun `{self}` because the task is None (deleted)") + return None + else: + children_tasks = self.children.values_list('task', flat=True) + if None in children_tasks: + logger.error(f"Cannot rerun `{self}` because one or more children submission tasks are None (deleted)") + return None + + # Create a new submission submission_arg_dict = { 'owner': self.owner, - 'task': task or self.task, + 'task': new_submission_task, 'phase': self.phase, 'data': self.data, 'has_children': self.has_children, - 'is_specific_task_re_run': bool(task), + 'is_specific_task_re_run': is_specific_task_re_run, 'fact_sheet_answers': self.fact_sheet_answers, } sub = Submission(**submission_arg_dict) sub.save(ignore_submission_limit=True) - # No need to rerun on children if this is running on a specific task - if not self.has_children or sub.is_specific_task_re_run: - self.refresh_from_db() + # set tasks for rerunning + if flag_rerun_specific_task_or_has_no_children: + # in case of a submission with no children or specific task rerun + # submission with no children is same as submission with one task tasks = [sub.task] else: + # in case submission has multiple children or multiple task rerun + # tasks are gathered from the children submissions tasks = Task.objects.filter(pk__in=self.children.values_list('task', flat=True)) + sub.start(tasks=tasks) return sub diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index 7b9aa43d7..e7a6f17ed 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -323,7 +323,11 @@ .fail(function (response) { if(response.responseJSON.detail){ toastr.error(response.responseJSON.detail) - } else { + } + else if(response.responseJSON.error_msg){ + toastr.error(response.responseJSON.error_msg) + } + else { toastr.error(response.responseText) } }) From cb7713756eba2ebd174f68c4b1bf3433162b0288 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 28 Jan 2024 16:14:00 +0500 Subject: [PATCH 087/136] warning message displayed on task deletion --- src/static/riot/tasks/management.tag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/static/riot/tasks/management.tag b/src/static/riot/tasks/management.tag index 06d24bb2f..6ae6e34ca 100644 --- a/src/static/riot/tasks/management.tag +++ b/src/static/riot/tasks/management.tag @@ -639,7 +639,7 @@ } self.delete_task = function (task) { - if (confirm("Are you sure you want to delete '" + task.name + "'?")) { + if (confirm("Are you sure you want to delete '" + task.name + "'?\nSubmissions using this task cannot rerun!")) { CODALAB.api.delete_task(task.id) .done(function () { self.update_tasks() @@ -654,7 +654,7 @@ } self.delete_tasks = function () { - if (confirm(`Are you sure you want to delete multiple tasks?`)) { + if (confirm(`Are you sure you want to delete multiple tasks?\nSubmissions using these tasks cannot rerun!`)) { CODALAB.api.delete_tasks(self.marked_tasks) .done(function () { self.update_tasks() From a9066d5444fd3f98886ab8f1af2ed227be0347d2 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 1 Feb 2024 16:00:23 +0500 Subject: [PATCH 088/136] added env option to disable and enable regular signin and sign up --- .env_sample | 6 ++++++ src/apps/profiles/urls_accounts.py | 4 ---- src/apps/profiles/views.py | 6 ++++++ src/settings/base.py | 7 +++++++ src/templates/base.html | 4 +++- src/templates/registration/login.html | 6 +++++- src/utils/context_processors.py | 2 ++ 7 files changed, 29 insertions(+), 6 deletions(-) diff --git a/.env_sample b/.env_sample index b39a83de8..b13fe50a4 100644 --- a/.env_sample +++ b/.env_sample @@ -70,6 +70,12 @@ AWS_QUERYSTRING_AUTH=False # ----------------------------------------------------------------------------- RERUN_SUBMISSION_LIMIT=30 + # ----------------------------------------------------------------------------- + # Enable or disbale regular email sign-in an sign-up + # ----------------------------------------------------------------------------- + ENABLE_SIGN_UP=True + ENABLE_SIGN_IN=True + # # S3 storage example # STORAGE_TYPE=s3 diff --git a/src/apps/profiles/urls_accounts.py b/src/apps/profiles/urls_accounts.py index 03acf8254..779292962 100644 --- a/src/apps/profiles/urls_accounts.py +++ b/src/apps/profiles/urls_accounts.py @@ -8,10 +8,6 @@ urlpatterns = [ url(r'^signup', views.sign_up, name="signup"), path('login/', views.log_in, name='login'), - # url(r'^user_profile', views.user_profile, name="user_profile"), - # path('login/', auth_views.LoginView.as_view(extra_context=extra_context), name='login'), - # path('login/', views.LoginView.as_view(), name='login'), - # path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('logout/', views.LogoutView.as_view(), name='logout'), path('password_reset/', views.CustomPasswordResetView.as_view(), name='password_reset'), path('password_reset/done/', auth_views.PasswordResetDoneView.as_view(), name='password_reset_done'), diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 33ab6235d..3b6b22169 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -104,6 +104,12 @@ def activateEmail(request, user, to_email): def sign_up(request): + + # If sign up is not enabled then redirect to login + # this is for security as some users may access sign up page using the url + if not settings.ENABLE_SIGN_UP: + return redirect('accounts:login') + context = {} context['chahub_signup_url'] = "{}/profiles/signup?next={}/social/login/chahub".format( settings.SOCIAL_AUTH_CHAHUB_BASE_URL, diff --git a/src/settings/base.py b/src/settings/base.py index d5047db82..b8d133a05 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -469,3 +469,10 @@ # on default queue when number of submissions are < RERUN_SUBMISSION_LIMIT # ============================================================================= RERUN_SUBMISSION_LIMIT = os.environ.get('RERUN_SUBMISSION_LIMIT', 30) + + +# ============================================================================= +# Enable or disbale regular email sign-in an sign-up +# ============================================================================= +ENABLE_SIGN_UP = os.environ.get('ENABLE_SIGN_UP', 'True').lower() == 'true' +ENABLE_SIGN_IN = os.environ.get('ENABLE_SIGN_IN', 'True').lower() == 'true' diff --git a/src/templates/base.html b/src/templates/base.html index 87ef997ed..fd54b5ab6 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -177,7 +177,9 @@ {% else %} Login - Sign-up + {% if ENABLE_SIGN_UP %} + Sign-up + {% endif %} {% endif %} diff --git a/src/templates/registration/login.html b/src/templates/registration/login.html index 55d8f10cf..9ffe94320 100644 --- a/src/templates/registration/login.html +++ b/src/templates/registration/login.html @@ -6,6 +6,7 @@

Login

+ {% if ENABLE_SIGN_IN %}
{% csrf_token %} @@ -52,11 +53,14 @@

-

New to us? Sign Up

+ {% if ENABLE_SIGN_UP %} +

Don't have an account? Sign Up

+ {% endif %}

Forgot your password?

+ {% endif %} {% endblock %} \ No newline at end of file diff --git a/src/utils/context_processors.py b/src/utils/context_processors.py index 4b6d77fab..dd49ccbbb 100644 --- a/src/utils/context_processors.py +++ b/src/utils/context_processors.py @@ -23,4 +23,6 @@ def common_settings(request): 'USER_JSON_DATA': json.dumps(user_json_data), 'RABBITMQ_MANAGEMENT_URL': f"http://{settings.DOMAIN_NAME}:{settings.RABBITMQ_MANAGEMENT_PORT}", 'FLOWER_URL': f"http://{settings.DOMAIN_NAME}:{settings.FLOWER_PUBLIC_PORT}", + 'ENABLE_SIGN_UP': settings.ENABLE_SIGN_UP, + 'ENABLE_SIGN_IN': settings.ENABLE_SIGN_IN, } From f1dd8aeda078902b6f88564ed6e5355000f2139d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sat, 3 Feb 2024 12:50:25 +0500 Subject: [PATCH 089/136] login html hide `login` title when sign in not enabled --- src/templates/registration/login.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/registration/login.html b/src/templates/registration/login.html index 9ffe94320..2f1372819 100644 --- a/src/templates/registration/login.html +++ b/src/templates/registration/login.html @@ -3,10 +3,10 @@ {% block content %}
+ {% if ENABLE_SIGN_IN %}

Login

- {% if ENABLE_SIGN_IN %}
{% csrf_token %} @@ -63,4 +63,4 @@

{% endif %}
-{% endblock %} \ No newline at end of file +{% endblock %} From 95bf70c15f54184255f1d114ccc3c42d7acc10a5 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 7 Feb 2024 14:42:02 +0500 Subject: [PATCH 090/136] organization oidc login added --- src/apps/oidc_configurations/__init__.py | 0 src/apps/oidc_configurations/admin.py | 6 + src/apps/oidc_configurations/apps.py | 5 + .../migrations/0001_initial.py | 29 ++++ .../migrations/__init__.py | 0 src/apps/oidc_configurations/models.py | 14 ++ src/apps/oidc_configurations/tests.py | 3 + src/apps/oidc_configurations/urls.py | 10 ++ src/apps/oidc_configurations/views.py | 144 ++++++++++++++++++ src/apps/profiles/views.py | 6 + src/settings/base.py | 1 + src/templates/oidc/oidc_complete.html | 16 ++ src/templates/registration/login.html | 22 +++ src/urls.py | 2 + 14 files changed, 258 insertions(+) create mode 100644 src/apps/oidc_configurations/__init__.py create mode 100644 src/apps/oidc_configurations/admin.py create mode 100644 src/apps/oidc_configurations/apps.py create mode 100644 src/apps/oidc_configurations/migrations/0001_initial.py create mode 100644 src/apps/oidc_configurations/migrations/__init__.py create mode 100644 src/apps/oidc_configurations/models.py create mode 100644 src/apps/oidc_configurations/tests.py create mode 100644 src/apps/oidc_configurations/urls.py create mode 100644 src/apps/oidc_configurations/views.py create mode 100644 src/templates/oidc/oidc_complete.html diff --git a/src/apps/oidc_configurations/__init__.py b/src/apps/oidc_configurations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/apps/oidc_configurations/admin.py b/src/apps/oidc_configurations/admin.py new file mode 100644 index 000000000..5ea6e683f --- /dev/null +++ b/src/apps/oidc_configurations/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin +from .models import Auth_Organization + +admin.site.register(Auth_Organization) + +# Register your models here. diff --git a/src/apps/oidc_configurations/apps.py b/src/apps/oidc_configurations/apps.py new file mode 100644 index 000000000..3d757062b --- /dev/null +++ b/src/apps/oidc_configurations/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OidcConfigurationsConfig(AppConfig): + name = 'oidc_configurations' diff --git a/src/apps/oidc_configurations/migrations/0001_initial.py b/src/apps/oidc_configurations/migrations/0001_initial.py new file mode 100644 index 000000000..266976235 --- /dev/null +++ b/src/apps/oidc_configurations/migrations/0001_initial.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.17 on 2024-02-04 14:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Auth_Organization', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('client_id', models.CharField(max_length=255)), + ('client_secret', models.CharField(max_length=255)), + ('authorization_url', models.URLField()), + ('token_url', models.URLField()), + ('user_info_url', models.URLField()), + ('redirect_url', models.URLField()), + ('button_bg_color', models.CharField(default='#2C3E4C', max_length=20)), + ('button_text_color', models.CharField(default='#FFFFFF', max_length=20)), + ], + ), + ] diff --git a/src/apps/oidc_configurations/migrations/__init__.py b/src/apps/oidc_configurations/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/apps/oidc_configurations/models.py b/src/apps/oidc_configurations/models.py new file mode 100644 index 000000000..07525ca3c --- /dev/null +++ b/src/apps/oidc_configurations/models.py @@ -0,0 +1,14 @@ +# oidc_configurations/models.py +from django.db import models + + +class Auth_Organization(models.Model): + name = models.CharField(max_length=255) + client_id = models.CharField(max_length=255) + client_secret = models.CharField(max_length=255) + authorization_url = models.URLField() + token_url = models.URLField() + user_info_url = models.URLField() + redirect_url = models.URLField() + button_bg_color = models.CharField(max_length=20, default='#2C3E4C') + button_text_color = models.CharField(max_length=20, default='#FFFFFF') diff --git a/src/apps/oidc_configurations/tests.py b/src/apps/oidc_configurations/tests.py new file mode 100644 index 000000000..7ce503c2d --- /dev/null +++ b/src/apps/oidc_configurations/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/src/apps/oidc_configurations/urls.py b/src/apps/oidc_configurations/urls.py new file mode 100644 index 000000000..7bfae4f99 --- /dev/null +++ b/src/apps/oidc_configurations/urls.py @@ -0,0 +1,10 @@ +# oidc_configurations/urls.py +from django.urls import path +from .views import organization_oidc_login, oidc_complete + +app_name = 'oidc_configurations' + +urlpatterns = [ + path('organization_oidc_login/', organization_oidc_login, name='organization_oidc_login'), + path('complete//', oidc_complete, name='oidc_complete'), +] diff --git a/src/apps/oidc_configurations/views.py b/src/apps/oidc_configurations/views.py new file mode 100644 index 000000000..63f59989e --- /dev/null +++ b/src/apps/oidc_configurations/views.py @@ -0,0 +1,144 @@ +# oidc_configurations/views.py +import base64 +import requests +from django.shortcuts import render, redirect, get_object_or_404 +from .models import Auth_Organization + + +def organization_oidc_login(request): + # Check if this is a post request and it contains organization_oauth2_login + if request.method == 'POST' and 'organization_oidc_login' in request.POST: + # Get auth organization id from the request + auth_organization_id = request.POST.get('organization_oidc_login') + + # Get auth organization using its id + organization = get_object_or_404(Auth_Organization, pk=auth_organization_id) + + if organization: + # Create a redirect url consisiting of + # - authorization_url + # - client_id + # - response_type + # - scope + # - redirect_uri + oidc_auth_url = ( + f"{organization.authorization_url}?" + f"client_id={organization.client_id}&" + f"response_type=code&" + "scope=openid profile email&" + f"redirect_uri={organization.redirect_url}" + ) + + # Redirect the user to the OAuth2 provider's authorization URL + return redirect(oidc_auth_url) + + # Handle other cases or render a different template if needed + return render(request, 'registration/login.html') + + +def oidc_complete(request, auth_organization_id): + + # create empty context + context = {} + + # Get error or authorization code from the query string + error = request.GET.get('error', None) + error_description = request.GET.get('error_description', None) + authorization_code = request.GET.get('code', None) + + if error: + context["error"] = error + + if error_description: + context["error_description"] = error_description + + # Token exhange process + if authorization_code: + + print(f"\n\n\n Authentication Code: {authorization_code} \n\n\n") + + try: + # STEP 1: Get auth organization using its id + organization = get_object_or_404(Auth_Organization, pk=auth_organization_id) + if organization: + + # Get access token + access_token, token_error = get_access_token(organization, authorization_code) + if token_error: + context["error"] = token_error + + print(f"\n\n\n Access Token: {access_token} \n\n\n") + + # STEP 2: Make a POST request to the user info endpoint to get user info + user_info, user_info_error = get_user_info(organization, access_token) + if user_info_error: + context["error"] = user_info_error + + print(f"\n\n\n User Info: {user_info} \n\n\n") + + print(user_info) + # STEP 3: Check in db if this user exists then login, if user is new create a new user and then login + + else: + context["error"] = "Invalid Organization ID!" + except Exception as e: + context["error"] = f"{e}" + + return render(request, 'oidc/oidc_complete.html', context) + + +def get_access_token(organization, authorization_code): + + token_url = organization.token_url + client_id = organization.client_id + client_secret = organization.client_secret + redirect_url = organization.redirect_url + + auth_header = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode("utf-8") + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": f"Basic {auth_header}", + } + data = { + "grant_type": "authorization_code", + "code": authorization_code, + "redirect_uri": redirect_url, + } + + # response = requests.post(token_url, data=data, headers=headers) + + # try: + # token_data = response.json() + # return token_data.get('access_token'), None + # except Exception as e: + # return None, e + + try: + response = requests.post(token_url, data=data, headers=headers) + response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx) + token_data = response.json() + access_token = token_data.get('access_token') + return access_token, None + except requests.exceptions.RequestException as e: + print(f"Error during token request: {e}") + return None, e + except Exception as e: + print(f"Error parsing token response: {e}") + return None, e + + +def get_user_info(organization, access_token): + + user_info_url = organization.user_info_url + + headers = { + 'Authorization': f'Bearer {access_token}', + } + + response = requests.get(user_info_url, headers=headers) + + try: + user_info = response.json() + return user_info, None + except Exception as e: + return None, e diff --git a/src/apps/profiles/views.py b/src/apps/profiles/views.py index 33ab6235d..fda7e124a 100644 --- a/src/apps/profiles/views.py +++ b/src/apps/profiles/views.py @@ -21,6 +21,7 @@ UserNotificationSerializer from .forms import SignUpForm, LoginForm from .models import User, Organization, Membership +from oidc_configurations.models import Auth_Organization from .tokens import account_activation_token @@ -172,6 +173,11 @@ def log_in(request): else: context['form'] = form + # Fetch auth_organizations from the database + auth_organizations = Auth_Organization.objects.all() + if auth_organizations: + context['auth_organizations'] = auth_organizations + if not context.get('form'): context['form'] = LoginForm() return render(request, 'registration/login.html', context) diff --git a/src/settings/base.py b/src/settings/base.py index d5047db82..513ae7918 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -60,6 +60,7 @@ 'health', 'forums', 'announcements', + 'oidc_configurations', ) INSTALLED_APPS = THIRD_PARTY_APPS + OUR_APPS diff --git a/src/templates/oidc/oidc_complete.html b/src/templates/oidc/oidc_complete.html new file mode 100644 index 000000000..293d52823 --- /dev/null +++ b/src/templates/oidc/oidc_complete.html @@ -0,0 +1,16 @@ +{% extends 'base.html' %} +{% load static %} + +{% block content %} +
+ {% if error %} +

OIDC Error

+
+

{{ error }}

+ {% if error_description %} +

{{ error_description }}

+ {% endif %} +
+ {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/src/templates/registration/login.html b/src/templates/registration/login.html index 55d8f10cf..99bf8033c 100644 --- a/src/templates/registration/login.html +++ b/src/templates/registration/login.html @@ -58,5 +58,27 @@

+ + + {% if auth_organizations %} +
+

Organization Login

+ {% for organization in auth_organizations %} +
+ {% csrf_token %} + +
+ {% endfor %} + +
+ {% endif %} {% endblock %} \ No newline at end of file diff --git a/src/urls.py b/src/urls.py index ea610fd46..a74ff4b5b 100644 --- a/src/urls.py +++ b/src/urls.py @@ -30,6 +30,8 @@ path('accounts/', include('profiles.urls_accounts')), path('admin/', admin.site.urls), path('social/', include('social_django.urls', namespace='social')), + path('oidc/', include('oidc_configurations.urls')), + ] From 93a8af6d8da3eaef615bff1560a5c39981d372f9 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 7 Feb 2024 14:48:01 +0500 Subject: [PATCH 091/136] unused test file removed --- src/apps/oidc_configurations/tests.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 src/apps/oidc_configurations/tests.py diff --git a/src/apps/oidc_configurations/tests.py b/src/apps/oidc_configurations/tests.py deleted file mode 100644 index 7ce503c2d..000000000 --- a/src/apps/oidc_configurations/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. From 7a3f8ad454a3bb86fd56a7ba8faf62825221e01b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 7 Feb 2024 15:14:18 +0500 Subject: [PATCH 092/136] files tab: show login if not loggedin + hide available column from non-organizers --- src/static/riot/competitions/detail/_tabs.tag | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/static/riot/competitions/detail/_tabs.tag b/src/static/riot/competitions/detail/_tabs.tag index 75bceec2a..1a9769790 100644 --- a/src/static/riot/competitions/detail/_tabs.tag +++ b/src/static/riot/competitions/detail/_tabs.tag @@ -50,17 +50,25 @@
- + +
- + +
+ Log In or + Sign Up to view availbale files. +
+ + +
- - From dd3f738ebeb46bf6c09385785d8b1bba2aaa8ed9 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 7 Feb 2024 20:49:40 +0500 Subject: [PATCH 093/136] do not delete original submission when are reun is deleted, similarly do not delete rerun submission when original is deleted --- src/apps/competitions/models.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index 4b87ec35a..f3bc3823d 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -500,10 +500,17 @@ def __str__(self): return f"{self.phase.competition.title} submission PK={self.pk} by {self.owner.username}" def delete(self, **kwargs): + + # Check if any other submissions are using the same data + other_submissions_using_data = Submission.objects.filter(data=self.data).exclude(pk=self.pk).exists() + + if not other_submissions_using_data: + # If no other submissions are using the same data, delete it + self.data.delete() + # Also clean up details on delete self.details.all().delete() - # Call this here so that the data_file for the submission also gets deleted from storage - self.data.delete() + super().delete(**kwargs) def save(self, ignore_submission_limit=False, **kwargs): From 4004cec2f146e7c2b9a7fcb9deec4e5ceb23581f Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 8 Feb 2024 14:34:51 +0500 Subject: [PATCH 094/136] http client --- src/apps/oidc_configurations/views.py | 40 ++++++++++++++++++++------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/apps/oidc_configurations/views.py b/src/apps/oidc_configurations/views.py index 63f59989e..c6a302cfa 100644 --- a/src/apps/oidc_configurations/views.py +++ b/src/apps/oidc_configurations/views.py @@ -1,6 +1,8 @@ # oidc_configurations/views.py import base64 +import http.client import requests +from urllib.parse import urlparse from django.shortcuts import render, redirect, get_object_or_404 from .models import Auth_Organization @@ -19,17 +21,15 @@ def organization_oidc_login(request): # - authorization_url # - client_id # - response_type - # - scope # - redirect_uri oidc_auth_url = ( f"{organization.authorization_url}?" f"client_id={organization.client_id}&" f"response_type=code&" - "scope=openid profile email&" f"redirect_uri={organization.redirect_url}" ) - # Redirect the user to the OAuth2 provider's authorization URL + # Redirect the user to the OIDC provider's authorization URL return redirect(oidc_auth_url) # Handle other cases or render a different template if needed @@ -104,6 +104,9 @@ def get_access_token(organization, authorization_code): "code": authorization_code, "redirect_uri": redirect_url, } + print("token url: ", token_url) + print("data: ", data) + print("header: ", headers) # response = requests.post(token_url, data=data, headers=headers) @@ -113,17 +116,34 @@ def get_access_token(organization, authorization_code): # except Exception as e: # return None, e + + # try: + # response = requests.request("POST", token_url, data=data, headers=headers) + # response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx) + # token_data = response.json() + # access_token = token_data.get('access_token') + # return access_token, None + # except requests.exceptions.RequestException as e: + # print(f"Error during token request: {e}") + # return None, e + # except Exception as e: + # print(f"Error parsing token response: {e}") + # return None, e + try: - response = requests.post(token_url, data=data, headers=headers) - response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx) - token_data = response.json() + parsed_url = urlparse(token_url) + conn = http.client.HTTPConnection(parsed_url.hostname, parsed_url.port) + conn.request("POST", parsed_url.path, data, headers) + response = conn.getresponse() + token_data = response.read().decode("utf-8") access_token = token_data.get('access_token') + conn.close() + print("Response:", token_data) + # Parse token_data if needed + # access_token = ... return access_token, None - except requests.exceptions.RequestException as e: - print(f"Error during token request: {e}") - return None, e except Exception as e: - print(f"Error parsing token response: {e}") + print(f"Error during token request: {e}") return None, e From ba021c37a8e71f7d69c45b61a14429dcfbeeb6d6 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 8 Feb 2024 16:48:24 +0500 Subject: [PATCH 095/136] less than and greater than rendering fixed --- src/static/js/ours/latex_markdown_html.js | 49 ++++++++++++----------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/static/js/ours/latex_markdown_html.js b/src/static/js/ours/latex_markdown_html.js index 4a0e645c2..6ee169f08 100644 --- a/src/static/js/ours/latex_markdown_html.js +++ b/src/static/js/ours/latex_markdown_html.js @@ -1,28 +1,29 @@ // Function to render Markdown, HTML and Latex and return updated content function renderMarkdownWithLatex(content) { - if(content === null){ - return [] - } - const parsedHtml = new DOMParser().parseFromString(marked(content), "text/html") - - const traverseAndRenderLatex = (node) => { - if (node.nodeType === Node.ELEMENT_NODE) { - const latexPattern = /\$\$([\s\S]*?)\$\$|\$([^\$\n]*?)\$/g - const hasLatex = latexPattern.test(node.textContent) - if (hasLatex) { - const tempDiv = document.createElement('div') - tempDiv.innerHTML = node.innerHTML.replace(latexPattern, (_, formula1, formula2) => { - const formula = formula1 || formula2 - return katex.renderToString(formula, { throwOnError: false }) - }); - node.innerHTML = tempDiv.innerHTML - } + if(content === null){ + return [] + } + const parsedHtml = new DOMParser().parseFromString(marked(content), "text/html") + + const traverseAndRenderLatex = (node) => { + if (node.nodeType === Node.ELEMENT_NODE) { + const latexPattern = /\$\$([\s\S]*?)\$\$|\$([^\$\n]*?)\$/g + const hasLatex = latexPattern.test(node.textContent) + if (hasLatex) { + const tempDiv = document.createElement('div') + tempDiv.innerHTML = node.innerHTML.replace(latexPattern, (_, formula1, formula2) => { + const formula = formula1 || formula2 + const decodedFormula = formula.replace(/</g, '<').replace(/>/g, '>') + return katex.renderToString(decodedFormula, { throwOnError: false }) + }); + node.innerHTML = tempDiv.innerHTML } - - node.childNodes.forEach(traverseAndRenderLatex) - }; - - traverseAndRenderLatex(parsedHtml.body) - - return parsedHtml.body.childNodes + } + + node.childNodes.forEach(traverseAndRenderLatex) + }; + + traverseAndRenderLatex(parsedHtml.body) + + return parsedHtml.body.childNodes } \ No newline at end of file From a40e4e414cb0890b08d43a9d77314ba1ad95e469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Thu, 8 Feb 2024 16:40:53 +0100 Subject: [PATCH 096/136] Add an example of competition docker image --- src/static/riot/competitions/editor/_competition_details.tag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag index 50f301342..c1121e555 100644 --- a/src/static/riot/competitions/editor/_competition_details.tag +++ b/src/static/riot/competitions/editor/_competition_details.tag @@ -109,7 +109,7 @@
- +
From dbe8f8817735b6348e673b6aca013ed58431b2ad Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 9 Feb 2024 15:22:08 +0500 Subject: [PATCH 097/136] warning message added --- src/static/riot/tasks/management.tag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/static/riot/tasks/management.tag b/src/static/riot/tasks/management.tag index 6ae6e34ca..92e2eb506 100644 --- a/src/static/riot/tasks/management.tag +++ b/src/static/riot/tasks/management.tag @@ -639,7 +639,7 @@ } self.delete_task = function (task) { - if (confirm("Are you sure you want to delete '" + task.name + "'?\nSubmissions using this task cannot rerun!")) { + if (confirm("Are you sure you want to delete '" + task.name + "'?\nSubmissions using this task cannot rerun! Results displayed on leaderboard can also be affected!")) { CODALAB.api.delete_task(task.id) .done(function () { self.update_tasks() @@ -654,7 +654,7 @@ } self.delete_tasks = function () { - if (confirm(`Are you sure you want to delete multiple tasks?\nSubmissions using these tasks cannot rerun!`)) { + if (confirm(`Are you sure you want to delete multiple tasks?\nSubmissions using these tasks cannot rerun! Results displayed on leaderboard can also be affected!`)) { CODALAB.api.delete_tasks(self.marked_tasks) .done(function () { self.update_tasks() From ac48c17c300aa5dcce4ddc4a5c4bb1c4f3449bc8 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 9 Feb 2024 16:43:38 +0500 Subject: [PATCH 098/136] show competiitons using the queue --- src/apps/api/serializers/queues.py | 25 ++++++++++++++++++++++++- src/static/riot/queues/management.tag | 19 ++++++++++++++----- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/apps/api/serializers/queues.py b/src/apps/api/serializers/queues.py index deea441d0..3614b61a2 100644 --- a/src/apps/api/serializers/queues.py +++ b/src/apps/api/serializers/queues.py @@ -3,8 +3,8 @@ from api.mixins import DefaultUserCreateMixin from queues.models import Queue - from profiles.models import User +from django.db.models import Q class OrganizerSerializer(serializers.ModelSerializer): @@ -61,6 +61,7 @@ class QueueSerializer(QueueOwnerMixin, serializers.ModelSerializer): is_owner = serializers.SerializerMethodField() owner = serializers.CharField(source='owner.username', read_only=True) organizers = OrganizerSerializer(many=True, read_only=True) + competitions = serializers.SerializerMethodField() class Meta: model = Queue @@ -74,6 +75,7 @@ class Meta: 'created_when', 'is_owner', 'id', + 'competitions', ) # This serializer is read only, basically.. read_only_fields = ( @@ -85,4 +87,25 @@ class Meta: 'broker_url', 'created_when', 'is_owner', + 'competitions', ) + + def get_competitions(self, obj): + # get user from the context request + user = self.context['request'].user + + # for super user return all competiitons using this queue + # for admin return competitions where this user is organizer using this queue + # for non-admin return public competitions using this queue + if user.is_superuser: + # Fetch all competitions + competitions = obj.competitions.all().values('id', 'title') + else: + # Fetch all competitions where user is organizer or competition is published + competitions = obj.competitions.filter( + Q(published=True) | + Q(created_by=user) | + Q(collaborators=user) + ).values('id', 'title') + + return competitions diff --git a/src/static/riot/queues/management.tag b/src/static/riot/queues/management.tag index be74a5132..ba138d6d9 100644 --- a/src/static/riot/queues/management.tag +++ b/src/static/riot/queues/management.tag @@ -133,11 +133,20 @@

Broker URL:

-
- -
-

Vhost

- {selected_queue.vhost} + {selected_queue.broker_url} + +

Vhost:

+ {selected_queue.vhost} + + +

Competitions using this queue:

+ +
Close
From 4f441b6f80c15aee0aa914670d1f611fd5907048 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sat, 10 Feb 2024 00:01:46 +0500 Subject: [PATCH 099/136] Create Codabench Statistics (#1307) * generate codabench statistics * added comments, removed extra code, handled None values * flake * reward cleaned --- src/apps/competitions/statistics.py | 111 ++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/apps/competitions/statistics.py diff --git a/src/apps/competitions/statistics.py b/src/apps/competitions/statistics.py new file mode 100644 index 000000000..d77cc624a --- /dev/null +++ b/src/apps/competitions/statistics.py @@ -0,0 +1,111 @@ +# -------------------------------------------------- +# Imports +# -------------------------------------------------- +import os +from competitions.models import Competition + +# -------------------------------------------------- +# Setting constants +# -------------------------------------------------- +BASE_URL = "https://www.codabench.org/competitions/" +STATISTICS_DIR = "/app/statistics/" +CSV_FILE_NAME = "codabench_competition_statistics.csv" +CSV_PATH = STATISTICS_DIR + CSV_FILE_NAME + + +def create_codabench_statistics(): + """ + This function prepares a CSV file with all published competitions + """ + + # Create statistics directory if not already createad + if not os.path.exists(STATISTICS_DIR): + os.makedirs(STATISTICS_DIR) + + # Write header of the CSV file + with open(CSV_PATH, 'w', newline='') as output_file: + # Header for the csv + header = 'title; description; participants; submissions; year; phases; reward; duration (days); url;\n' + output_file.write(header) + + # loop over published competitions + for comp in Competition.objects.filter(published=True): + + # get title + title = comp.title + title = clean_string(title) + + # get description + desc = comp.description + desc = clean_string(desc) + + # get participants + num_participants = comp.participants.count() + + # get phases + phases = comp.phases.all() + num_phases = len(phases) + + # get submissions + num_submissions = 0 + for phase in phases: + num_submissions += phase.submissions.count() + + # get competition first phase year + year = phases[0].start.year + + # get competition start and end date + start_date = phases[0].start + end_date = phases[num_phases - 1].end + # if last phase has no end date, set end date to last phase start date + if end_date is None: + end_date = phases[num_phases - 1].start + + # compute duration of the competition + duration = (end_date - start_date).days + + # get reward + reward = comp.reward + # set reward to empty string if none + if reward is None: + reward = "" + else: + reward = clean_string(reward) + + # prepare competition url + url = f"{BASE_URL}{comp.id}" + + # prepare a row with all the computed information for one competition + row = '{}; {}; {}; {}; {}; {}; {}; {}; {}; \n'.format( + title, + desc, + num_participants, + num_submissions, + year, + num_phases, + reward, + duration, + url + ) + + # write row in the CSV file + with open(CSV_PATH, 'a') as output_file: + output_file.write(row) + + +def clean_string(text): + """ + This function cleans an input text + """ + if ";" in text: + text = text.replace(";", ",") + + if '\n' in text: + text = text.replace(r'\n', ' ') + + if '\r' in text: + text = text.replace(r'\r', ' ') + + text = ''.join(text.splitlines()) + + return text From 61b0149066006d5b27db4fd182dfea4fe7899199 Mon Sep 17 00:00:00 2001 From: Benjamin Bearce Date: Fri, 9 Feb 2024 14:51:43 -0500 Subject: [PATCH 100/136] Logo Icons too big fix (Issue: 1290) (#1306) Logo Icons too big fix (Issue: 1290) (#1306) --- src/apps/api/serializers/competitions.py | 2 ++ .../migrations/0045_auto_20240129_2314.py | 19 +++++++++++ src/apps/competitions/models.py | 34 +++++++++++++++++++ src/static/riot/competitions/public-list.tag | 2 +- src/utils/data.py | 30 ++++++++++------ 5 files changed, 75 insertions(+), 12 deletions(-) create mode 100644 src/apps/competitions/migrations/0045_auto_20240129_2314.py diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index 2d1e1639c..e3bc20fb7 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -327,6 +327,7 @@ class CompetitionCreateSerializer(CompetitionSerializer): class CompetitionDetailSerializer(serializers.ModelSerializer): created_by = serializers.CharField(source='created_by.username', read_only=True) + logo_icon = NamedBase64ImageField(allow_null=True) pages = PageSerializer(many=True) phases = PhaseDetailSerializer(many=True) leaderboards = serializers.SerializerMethodField() @@ -347,6 +348,7 @@ class Meta: 'created_by', 'created_when', 'logo', + 'logo_icon', 'terms', 'pages', 'phases', diff --git a/src/apps/competitions/migrations/0045_auto_20240129_2314.py b/src/apps/competitions/migrations/0045_auto_20240129_2314.py new file mode 100644 index 000000000..2cb752724 --- /dev/null +++ b/src/apps/competitions/migrations/0045_auto_20240129_2314.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.17 on 2024-01-29 23:14 + +from django.db import migrations, models +import utils.data + + +class Migration(migrations.Migration): + + dependencies = [ + ('competitions', '0044_merge_20231221_1416'), + ] + + operations = [ + migrations.AddField( + model_name='competition', + name='logo_icon', + field=models.ImageField(blank=True, null=True, upload_to=utils.data.PathWrapper('logos', manual_override=True)), + ), + ] diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py index dc8e79b9a..1f96cc84e 100644 --- a/src/apps/competitions/models.py +++ b/src/apps/competitions/models.py @@ -1,9 +1,12 @@ import logging import uuid +import os +import io from django.conf import settings from django.contrib.sites.models import Site from django.contrib.postgres.fields import JSONField +from django.core.files.base import ContentFile from django.db import models from django.db.models import Q from django.urls import reverse @@ -15,6 +18,7 @@ from profiles.models import User, Organization from utils.data import PathWrapper from utils.storage import BundleStorage +from PIL import Image from tasks.models import Task @@ -32,6 +36,7 @@ class Competition(ChaHubSaveMixin, models.Model): title = models.CharField(max_length=256) logo = models.ImageField(upload_to=PathWrapper('logos'), null=True, blank=True) + logo_icon = models.ImageField(upload_to=PathWrapper('logos', manual_override=True), null=True, blank=True) created_by = models.ForeignKey(settings.AUTH_USER_MODEL, null=True, blank=True, on_delete=models.SET_NULL, related_name="competitions") created_when = models.DateTimeField(default=now) @@ -214,8 +219,37 @@ def get_chahub_data(self): return self.clean_private_data(data) + def make_logo_icon(self): + if self.logo: + # Read the content of the logo file + self.logo.name + self.logo_icon + icon_dirname_only = os.path.dirname(self.logo.name) # Get just the path + icon_basename_only = os.path.basename(self.logo.name) # Get just the filename + file_name = os.path.splitext(icon_basename_only)[0] + ext = os.path.splitext(icon_basename_only)[1] + new_path = os.path.join(icon_dirname_only, f"{file_name}_icon{ext}") + logo_content = self.logo.read() + original_logo = Image.open(io.BytesIO(logo_content)) + # Resize the image to a smaller size for logo_icon + width, height = original_logo.size + new_width = 100 # Specify the desired width for the logo_icon + new_height = int((new_width / width) * height) + resized_logo = original_logo.resize((new_width, new_height)) + # Create a BytesIO object to save the resized image + icon_content = io.BytesIO() + resized_logo.save(icon_content, format='PNG') + # Save the resized logo as logo_icon + self.logo_icon.save(new_path, ContentFile(icon_content.getvalue()), save=False) + def save(self, *args, **kwargs): super().save(*args, **kwargs) + if not self.logo: + pass + elif not self.logo_icon: + self.make_logo_icon() + elif os.path.dirname(self.logo.name) != os.path.dirname(self.logo_icon.name): + self.make_logo_icon() to_create = User.objects.filter( Q(id=self.created_by_id) | Q(id__in=self.collaborators.all().values_list('id', flat=True)) ).exclude(id__in=self.participants.values_list('user_id', flat=True)).distinct() diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index 2eb854a08..c9c8e4680 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -13,7 +13,7 @@
- +
diff --git a/src/utils/data.py b/src/utils/data.py index 0303888d6..7d743f361 100644 --- a/src/utils/data.py +++ b/src/utils/data.py @@ -19,20 +19,28 @@ class PathWrapper(object): """Helper to generate UUID's in file names while maintaining their extension""" - def __init__(self, base_directory): + def __init__(self, base_directory, manual_override=False): self.path = base_directory + self.manual_override = manual_override def __call__(self, instance, filename): - name, extension = os.path.splitext(filename) - truncated_uuid = uuid.uuid4().hex[0:12] - truncated_name = name[0:35] - - return os.path.join( - self.path, - now().strftime('%Y-%m-%d-%s'), - truncated_uuid, - "{0}{1}".format(truncated_name, extension) - ) + if not self.manual_override: + name, extension = os.path.splitext(filename) + truncated_uuid = uuid.uuid4().hex[0:12] + truncated_name = name[0:35] + + path = os.path.join( + self.path, + now().strftime('%Y-%m-%d-%s'), + truncated_uuid, + "{0}{1}".format(truncated_name, extension) + ) + else: + path = os.path.join( + filename + ) + + return path def make_url_sassy(path, permission='r', duration=60 * 60 * 24, content_type='application/zip'): From 62358ae2e823fe03a2e24bab0a4db50be6089581 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sat, 10 Feb 2024 12:06:51 +0500 Subject: [PATCH 101/136] public competitions page error fixed --- src/apps/api/serializers/queues.py | 31 ++++++++++++++++++++++++++++++ src/apps/api/views/queues.py | 4 ++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/apps/api/serializers/queues.py b/src/apps/api/serializers/queues.py index 3614b61a2..8b7e70090 100644 --- a/src/apps/api/serializers/queues.py +++ b/src/apps/api/serializers/queues.py @@ -61,6 +61,37 @@ class QueueSerializer(QueueOwnerMixin, serializers.ModelSerializer): is_owner = serializers.SerializerMethodField() owner = serializers.CharField(source='owner.username', read_only=True) organizers = OrganizerSerializer(many=True, read_only=True) + + class Meta: + model = Queue + fields = ( + 'name', + 'vhost', + 'is_public', + 'owner', + 'organizers', + 'broker_url', + 'created_when', + 'is_owner', + 'id', + ) + # This serializer is read only, basically.. + read_only_fields = ( + 'name', + 'vhost', + 'is_public', + 'owner', + 'organizers', + 'broker_url', + 'created_when', + 'is_owner', + ) + + +class QueueListSerializer(QueueOwnerMixin, serializers.ModelSerializer): + is_owner = serializers.SerializerMethodField() + owner = serializers.CharField(source='owner.username', read_only=True) + organizers = OrganizerSerializer(many=True, read_only=True) competitions = serializers.SerializerMethodField() class Meta: diff --git a/src/apps/api/views/queues.py b/src/apps/api/views/queues.py index db96188c6..23e486e85 100644 --- a/src/apps/api/views/queues.py +++ b/src/apps/api/views/queues.py @@ -13,7 +13,7 @@ class QueueViewSet(ModelViewSet): queryset = Queue.objects.all() - serializer_class = serializers.QueueSerializer + serializer_class = serializers.QueueListSerializer filter_fields = ('owner', 'is_public', 'name') filter_backends = (DjangoFilterBackend, SearchFilter) search_fields = ('name',) @@ -29,7 +29,7 @@ def get_queryset(self): def get_serializer_class(self): if self.request.method == 'GET': - return serializers.QueueSerializer + return serializers.QueueListSerializer else: return serializers.QueueCreationSerializer From 8f060153e60dbb916faac8e8c22767bf83e2c972 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sat, 10 Feb 2024 19:38:15 +0500 Subject: [PATCH 102/136] code simplified --- src/apps/api/serializers/queues.py | 33 +++--------------------------- 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/src/apps/api/serializers/queues.py b/src/apps/api/serializers/queues.py index 8b7e70090..b3d84aee6 100644 --- a/src/apps/api/serializers/queues.py +++ b/src/apps/api/serializers/queues.py @@ -88,38 +88,11 @@ class Meta: ) -class QueueListSerializer(QueueOwnerMixin, serializers.ModelSerializer): - is_owner = serializers.SerializerMethodField() - owner = serializers.CharField(source='owner.username', read_only=True) - organizers = OrganizerSerializer(many=True, read_only=True) +class QueueListSerializer(QueueSerializer): competitions = serializers.SerializerMethodField() - class Meta: - model = Queue - fields = ( - 'name', - 'vhost', - 'is_public', - 'owner', - 'organizers', - 'broker_url', - 'created_when', - 'is_owner', - 'id', - 'competitions', - ) - # This serializer is read only, basically.. - read_only_fields = ( - 'name', - 'vhost', - 'is_public', - 'owner', - 'organizers', - 'broker_url', - 'created_when', - 'is_owner', - 'competitions', - ) + class Meta(QueueSerializer.Meta): + fields = QueueSerializer.Meta.fields + ('competitions',) def get_competitions(self, obj): # get user from the context request From 2546aa0503e5c0b82967bbe9541e60610fb4f393 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 11 Feb 2024 15:09:37 +0500 Subject: [PATCH 103/136] rerun whole phase, only rerun submissions with no parent (this automatically runs parents and children submissions --- src/apps/api/views/competitions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 3dc2f0bba..ef86c11cc 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -634,8 +634,8 @@ def rerun_submissions(self, request, pk): phase = self.get_object() comp = phase.competition - # Get submissions - submissions = phase.submissions.all() + # Get submissions with no parent + submissions = phase.submissions.filter(parent__isnull=True) can_re_run_submissions = False error_message = "" From 5179ec32abb17789c09c239aa99383e5d1133f2b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 11 Feb 2024 19:37:14 +0500 Subject: [PATCH 104/136] add message in the empty error --- src/apps/api/views/submissions.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index f9582488d..f3d410792 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -209,9 +209,14 @@ def submission_leaderboard_connection(self, request, pk): submission = self.get_object() phase = submission.phase - if not (request.user.is_superuser or request.user == submission.owner): - if not phase.competition.collaborators.filter(pk=request.user.pk).exists(): - raise Http404 + # only super user, owner of submission and competition organizer can proceed + if not ( + request.user.is_superuser or + request.user == submission.owner or + request.user in phase.competition.all_organizers + ): + raise ValidationError("You cannot perform this action, contact the competition organizer!") + if submission.phase.leaderboard.submission_rule in Leaderboard.AUTO_SUBMISSION_RULES and not request.user.is_superuser: raise ValidationError("Users are not allowed to edit the leaderboard on this Competition") From 35eaaee6bf181cfdc904910ec66430b95164f4ea Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sun, 11 Feb 2024 19:41:57 +0500 Subject: [PATCH 105/136] unused import removed --- src/apps/api/views/submissions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index f3d410792..2a5278b97 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -6,7 +6,6 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework import status from rest_framework.decorators import api_view, permission_classes, action -from django.http import Http404 from rest_framework.exceptions import PermissionDenied, ValidationError from rest_framework.filters import SearchFilter from rest_framework.generics import get_object_or_404 From 3513e9efd745245605535e45bf6c4fe2408cc0a3 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Mon, 12 Feb 2024 12:24:04 +0500 Subject: [PATCH 106/136] test fixed, additional test added --- src/apps/api/views/submissions.py | 8 ++++++-- src/apps/competitions/tests/test_submissions.py | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 2a5278b97..2eb4a8cee 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -205,7 +205,11 @@ def has_admin_permission(self, user, submission): @action(detail=True, methods=('POST', 'DELETE')) def submission_leaderboard_connection(self, request, pk): + + # get submission submission = self.get_object() + + # get submission phase phase = submission.phase # only super user, owner of submission and competition organizer can proceed @@ -214,10 +218,10 @@ def submission_leaderboard_connection(self, request, pk): request.user == submission.owner or request.user in phase.competition.all_organizers ): - raise ValidationError("You cannot perform this action, contact the competition organizer!") + raise PermissionDenied("You cannot perform this action, contact the competition organizer!") if submission.phase.leaderboard.submission_rule in Leaderboard.AUTO_SUBMISSION_RULES and not request.user.is_superuser: - raise ValidationError("Users are not allowed to edit the leaderboard on this Competition") + raise PermissionDenied("Users are not allowed to edit the leaderboard on this Competition") if request.method == 'POST': # Removing any existing submissions on leaderboard unless multiples are allowed diff --git a/src/apps/competitions/tests/test_submissions.py b/src/apps/competitions/tests/test_submissions.py index 4e58ebd93..a7ae024f2 100644 --- a/src/apps/competitions/tests/test_submissions.py +++ b/src/apps/competitions/tests/test_submissions.py @@ -155,7 +155,21 @@ def test_only_owner_can_add_submission_to_leaderboard(self): self.client.force_login(different_user) url = reverse('submission-submission-leaderboard-connection', kwargs={'pk': parent_sub.pk}) resp = self.client.post(url) - assert resp.status_code == 404 + assert resp.status_code == 403 + assert resp.data["detail"] == "You cannot perform this action, contact the competition organizer!" + + def test_only_owner_can_remove_submission_from_leaderboard(self): + parent_sub = SubmissionFactory(has_children=True) + leaderboard = LeaderboardFactory() + parent_sub.phase.leaderboard = leaderboard + parent_sub.phase.save() + + different_user = UserFactory() + self.client.force_login(different_user) + url = reverse('submission-submission-leaderboard-connection', kwargs={'pk': parent_sub.pk}) + resp = self.client.delete(url) + assert resp.status_code == 403 + assert resp.data["detail"] == "You cannot perform this action, contact the competition organizer!" def test_adding_submission_removes_other_submissions_from_owner(self): leaderboard = LeaderboardFactory() From b1d549d166aea54e56a1416d0e5a224a2047b12b Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 14 Feb 2024 15:35:40 +0500 Subject: [PATCH 107/136] leaderboard number of enteries: exlude failed and cancelled submissions --- src/apps/api/views/competitions.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 3dc2f0bba..790b9066b 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -704,12 +704,18 @@ def get_leaderboard(self, request, pk): submission_detailed_results = {} for submission in query['submissions']: # count number of entries/number of submissions for the owner of this submission for this phase - # count all submissions with no parent and count all parents without counting the children + # count all submissions except: + # - child submissions (submissions who has a parent i.e. parent field is not null) + # - Failed submissions + # - Cancelled submissions num_entries = Submission.objects.filter( - Q(owner__username=submission['owner']) | Q(parent__owner__username=submission['owner']), + Q(owner__username=submission['owner']) | + Q(parent__owner__username=submission['owner']), phase=phase, ).exclude( - parent__isnull=False + Q(status=Submission.FAILED) | + Q(status=Submission.CANCELLED) | + Q(parent__isnull=False) ).count() submission_key = f"{submission['owner']}{submission['parent'] or submission['id']}" From 16579a7209a72ce2e482019cc3a63cfdefaf347d Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Thu, 15 Feb 2024 22:58:17 +0500 Subject: [PATCH 108/136] empty error is now shown to participant --- src/apps/api/views/submissions.py | 1 + src/static/riot/competitions/detail/submission_manager.tag | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 2eb4a8cee..394e160bb 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -220,6 +220,7 @@ def submission_leaderboard_connection(self, request, pk): ): raise PermissionDenied("You cannot perform this action, contact the competition organizer!") + # only super user and with these leaderboard rules (FORCE_LAST, FORCE_BEST, FORCE_LATEST_MULTIPLE) can proceed if submission.phase.leaderboard.submission_rule in Leaderboard.AUTO_SUBMISSION_RULES and not request.user.is_superuser: raise PermissionDenied("Users are not allowed to edit the leaderboard on this Competition") diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag index e7a6f17ed..95db0f866 100644 --- a/src/static/riot/competitions/detail/submission_manager.tag +++ b/src/static/riot/competitions/detail/submission_manager.tag @@ -262,7 +262,7 @@ CODALAB.events.trigger('submission_changed_on_leaderboard') }) .fail(function (response) { - toastr.error(response.responseJSON) + toastr.error(response.responseJSON.detail) }) event.stopPropagation() } @@ -273,7 +273,7 @@ CODALAB.events.trigger('submission_changed_on_leaderboard') }) .fail(function (response) { - toastr.error(response.responseJSON) + toastr.error(response.responseJSON.detail) }) event.stopPropagation() } From fca617e39daac215a1ffbf13fa8d55c6bcc564be Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 16 Feb 2024 12:21:23 +0500 Subject: [PATCH 109/136] empty error resolved --- src/apps/api/views/submissions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 394e160bb..2ce2d3fbe 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -237,7 +237,7 @@ def submission_leaderboard_connection(self, request, pk): if request.method == 'DELETE': if submission.phase.leaderboard.submission_rule not in [Leaderboard.ADD_DELETE, Leaderboard.ADD_DELETE_MULTIPLE]: - raise ValidationError("You are not allowed to remove a submission on this phase") + raise PermissionDenied("You are not allowed to remove a submission on this phase") submission.leaderboard = None submission.save() Submission.objects.filter(parent=submission).update(leaderboard=None) From 18f521bc65e180cf6f5946fa877d4a838337ae89 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 16 Feb 2024 14:43:01 +0500 Subject: [PATCH 110/136] small logo for popular and featured comps on home page --- src/apps/api/serializers/competitions.py | 1 + src/static/riot/competitions/tile/competition_tile.tag | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py index e3bc20fb7..7006529aa 100644 --- a/src/apps/api/serializers/competitions.py +++ b/src/apps/api/serializers/competitions.py @@ -404,6 +404,7 @@ class Meta: 'published', 'participant_count', 'logo', + 'logo_icon', 'description', 'competition_type', 'reward', diff --git a/src/static/riot/competitions/tile/competition_tile.tag b/src/static/riot/competitions/tile/competition_tile.tag index 5c4fdcb7e..59f48003e 100644 --- a/src/static/riot/competitions/tile/competition_tile.tag +++ b/src/static/riot/competitions/tile/competition_tile.tag @@ -2,7 +2,7 @@
- +
From 14901088bfe29870d10532cc704d8b821efc89ad Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Fri, 16 Feb 2024 17:27:39 +0500 Subject: [PATCH 111/136] privacy link updated --- src/templates/emails/base_email.html | 2 +- src/templates/emails/base_email.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/templates/emails/base_email.html b/src/templates/emails/base_email.html index 53b214c40..903e75e33 100644 --- a/src/templates/emails/base_email.html +++ b/src/templates/emails/base_email.html @@ -98,7 +98,7 @@

Hello{% if user %} {{ user.username }}{% endif %},

{% endif %} diff --git a/src/templates/emails/base_email.txt b/src/templates/emails/base_email.txt index 7048b2617..927e2db0c 100644 --- a/src/templates/emails/base_email.txt +++ b/src/templates/emails/base_email.txt @@ -14,4 +14,4 @@ Unsubscribe or manage notification settings: http://{{ site.domain }} Privacy policy: -http://codalab.github.io/codalab/notices.html +https://github.com/codalab/codalab-competitions/wiki/Privacy From 76e4d2b193c52e9e164758f98183e51c5b9993d9 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Sat, 17 Feb 2024 21:04:05 +0500 Subject: [PATCH 112/136] html tags in email text --- src/utils/email.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/email.py b/src/utils/email.py index ec557fdcf..846a5edbc 100644 --- a/src/utils/email.py +++ b/src/utils/email.py @@ -41,9 +41,10 @@ def sanitize(content): def codalab_send_markdown_email(subject, markdown_content, recipient_list, from_email=None): from_email = from_email if from_email else settings.DEFAULT_FROM_EMAIL - message = sanitize(markdown_content) - html_message = sanitize(markdown.markdown(message)) + html_message = markdown.markdown(markdown_content) + # message = sanitize(markdown_content) + # html_message = sanitize(markdown.markdown(message)) - message = EmailMultiAlternatives(subject, message, from_email=from_email, bcc=recipient_list) + message = EmailMultiAlternatives(subject, '', from_email=from_email, bcc=recipient_list) message.attach_alternative(html_message, 'text/html') message.send() From 0c2e12a54c130a562f66327ba68d630f6d02acd3 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 20 Feb 2024 14:28:15 +0500 Subject: [PATCH 113/136] some changes --- docker-compose.yml | 17 +++++++++++++++++ src/apps/oidc_configurations/views.py | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0b5c2c6ee..46516aa9f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -232,3 +232,20 @@ services: options: max-size: "20k" max-file: "10" + + + #----------------------------------------------- + # OIDC + #----------------------------------------------- + + oidc: + image: oidc_server + command: bash -c "cd /app/ && python manage.py runserver 0.0.0.0:9100" + ports: + - 9100:9100 + stdin_open: true + tty: true + logging: + options: + max-size: "20k" + max-file: "10" \ No newline at end of file diff --git a/src/apps/oidc_configurations/views.py b/src/apps/oidc_configurations/views.py index c6a302cfa..908b40987 100644 --- a/src/apps/oidc_configurations/views.py +++ b/src/apps/oidc_configurations/views.py @@ -25,7 +25,8 @@ def organization_oidc_login(request): oidc_auth_url = ( f"{organization.authorization_url}?" f"client_id={organization.client_id}&" - f"response_type=code&" + "response_type=code&" + "scope=openid profile email&" f"redirect_uri={organization.redirect_url}" ) From a3787e453ae730d376552b86971f45cf9531400e Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Tue, 20 Feb 2024 18:20:31 +0500 Subject: [PATCH 114/136] removed caching competitions for the public page --- src/apps/api/views/competitions.py | 1 - src/static/riot/competitions/public-list.tag | 24 ++++++++------------ 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/apps/api/views/competitions.py b/src/apps/api/views/competitions.py index 5e3a6aa43..a08aa6cdb 100644 --- a/src/apps/api/views/competitions.py +++ b/src/apps/api/views/competitions.py @@ -533,7 +533,6 @@ def create_dump(self, request, pk=None): serializer = CompetitionCreationTaskStatusSerializer({"status": "Success. Competition dump is being created."}) return Response(serializer.data, status=201) - @cache_response(key_func=DefaultListKeyConstructor()) @action(detail=False, methods=('GET',), pagination_class=LargePagination) def public(self, request): qs = self.get_queryset() diff --git a/src/static/riot/competitions/public-list.tag b/src/static/riot/competitions/public-list.tag index c9c8e4680..c2eef1f43 100644 --- a/src/static/riot/competitions/public-list.tag +++ b/src/static/riot/competitions/public-list.tag @@ -49,7 +49,6 @@
Download Phase Task TypeAvailable Available @@ -80,7 +88,7 @@ {file.task} {file.type} + {filesize(file.file_size * 1024)}