Competition Auto-Run Submissions

src/apps/api/serializers/competitions.py

@@ -245,6 +245,7 @@ class Meta:
             'registration_auto_approve',
             'queue',
             'enable_detailed_results',
+            'auto_run_submissions',
             'make_programs_available',
             'make_input_data_available',
             'docker_image',
@@ -359,6 +360,7 @@ class Meta:
             'submission_count',
             'queue',
             'enable_detailed_results',
+            'auto_run_submissions',
             'make_programs_available',
             'make_input_data_available',
             'docker_image',

src/apps/api/serializers/submissions.py

@@ -41,6 +41,7 @@ class SubmissionSerializer(serializers.ModelSerializer):
     on_leaderboard = serializers.BooleanField(read_only=True)
     task = TaskSerializer()
     created_when = serializers.DateTimeField(format="%Y-%m-%d %H:%M")
+    auto_run = serializers.SerializerMethodField(read_only=True)
 
     class Meta:
         model = Submission
@@ -66,6 +67,7 @@ class Meta:
             'leaderboard',
             'on_leaderboard',
             'task',
+            'auto_run'
         )
         read_only_fields = (
             'pk',
@@ -79,6 +81,10 @@ class Meta:
     def get_filename(self, instance):
         return basename(instance.data.data_file.name)
 
+    def get_auto_run(self, instance):
+        # returns this submission's competition auto_run_submissions Flag
+        return instance.phase.competition.auto_run_submissions
+
 
 class SubmissionLeaderBoardSerializer(serializers.ModelSerializer):
     scores = SubmissionScoreSerializer(many=True)
@@ -149,9 +155,12 @@ def get_filename(self, instance):
 
     def create(self, validated_data):
         tasks = validated_data.pop('tasks', None)
-
         sub = super().create(validated_data)
-        sub.start(tasks=tasks)
+
+        # Check if auto_run_submissions is enabled then run the submission
+        # Otherwise organizer will run manually
+        if sub.phase.competition.auto_run_submissions:
+            sub.start(tasks=tasks)
 
         return sub
 

src/apps/api/views/submissions.py

@@ -93,7 +93,7 @@ def check_object_permissions(self, request, obj):
 
             not_bot_user = self.request.user.is_authenticated and not self.request.user.is_bot
 
-            if self.action in ['update_fact_sheet', 're_run_submission']:
+            if self.action in ['update_fact_sheet', 'run_submission', 're_run_submission']:
                 # get_queryset will stop us from re-running something we're not supposed to
                 pass
             elif not self.request.user.is_authenticated or not_bot_user:
@@ -246,6 +246,21 @@ def cancel_submission(self, request, pk):
         canceled = submission.cancel()
         return Response({'canceled': canceled})
 
+    @action(detail=True, methods=('POST',))
+    def run_submission(self, request, pk):
+        submission = self.get_object()
+
+        # Only organizer of the competition can run the submission
+        if not self.has_admin_permission(request.user, submission):
+            raise PermissionDenied('You do not have permission to run this submission')
+
+        # Allow only to run a submission with status `Submitting`
+        if submission.status != Submission.SUBMITTING:
+            raise PermissionDenied('Cannot run a submission which is not in submitting status')
+
+        new_sub = submission.run()
+        return Response({'id': new_sub.id})
+
     @action(detail=True, methods=('POST',))
     def re_run_submission(self, request, pk):
         submission = self.get_object()

src/apps/competitions/migrations/0045_competition_auto_run_submissions.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.17 on 2024-01-22 10:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('competitions', '0044_merge_20231221_1416'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='competition',
+            name='auto_run_submissions',
+            field=models.BooleanField(default=True),
+        ),
+    ]

src/apps/competitions/models.py

@@ -60,6 +60,10 @@ class Competition(ChaHubSaveMixin, models.Model):
     reward = models.CharField(max_length=256, null=True, blank=True)
     report = models.CharField(max_length=256, null=True, blank=True)
 
+    # if true, submissions are auto-run when submitted
+    # if false, submissions run will be intiiated by organizer
+    auto_run_submissions = models.BooleanField(default=True)
+
     def __str__(self):
         return f"competition-{self.title}-{self.pk}-{self.competition_type}"
 
@@ -538,6 +542,13 @@ def start(self, tasks=None):
         from .tasks import run_submission
         run_submission(self.pk, tasks=tasks)
 
+    def run(self):
+        # get tasks from the phase
+        tasks = self.phase.tasks.all()
+        # start submission providing the tasks
+        self.start(tasks=tasks)
+        return self
+
     def re_run(self, task=None):
         submission_arg_dict = {
             'owner': self.owner,

src/apps/competitions/unpackers/v1.py

@@ -23,6 +23,7 @@ def __init__(self, *args, **kwargs):
             "description": self.competition_yaml.get("description", ""),
             "docker_image": docker_image,
             "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False),
+            "auto_run_submissions": self.competition_yaml.get('auto_run_submissions', True),
             "make_programs_available": self.competition_yaml.get('make_programs_available', False),
             "make_input_data_available": self.competition_yaml.get('make_input_data_available', False),
             "end_date": self.competition_yaml.get('end_date', None),

src/apps/competitions/unpackers/v2.py

@@ -14,6 +14,7 @@ def __init__(self, *args, **kwargs):
             "registration_auto_approve": self.competition_yaml.get('registration_auto_approve', False),
             "docker_image": self.competition_yaml.get('docker_image', 'codalab/codalab-legacy:py37'),
             "enable_detailed_results": self.competition_yaml.get('enable_detailed_results', False),
+            "auto_run_submissions": self.competition_yaml.get('auto_run_submissions', True),
             "make_programs_available": self.competition_yaml.get('make_programs_available', False),
             "make_input_data_available": self.competition_yaml.get('make_input_data_available', False),
             "description": self.competition_yaml.get("description", ""),

src/static/js/ours/client.js

@@ -102,6 +102,9 @@ CODALAB.api = {
     cancel_submission: function (id) {
         return CODALAB.api.request('GET', `${URLS.API}submissions/${id}/cancel_submission/`)
     },
+    run_submission: function (id) {
+        return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/run_submission/`)
+    },
     re_run_submission: function (id) {
         return CODALAB.api.request('POST', `${URLS.API}submissions/${id}/re_run_submission/`)
     },

src/static/riot/competitions/detail/submission_manager.tag

@@ -94,49 +94,57 @@
                 <sup data-tooltip="{submission.status_details}">
                     <i if="{submission.status === 'Failed'}" class="failed question circle icon"></i>
                 </sup>
+                <sup data-tooltip="An organizer will run your submission soon">
+                    <i if="{submission.status === 'Submitting' && !submission.auto_run}" class="question circle icon"></i>
+                </sup>
             </td>
             <td>{get_score(submission)}</td>
             <td class="center aligned">
                 <virtual if="{ opts.admin }">
-                    <span data-tooltip="Rerun Submission"
-                          data-inverted=""
-                          onclick="{ rerun_submission.bind(this, submission) }">
-                        <i class="icon blue redo"></i>
-                        <!-- rerun submission -->
+                    <!-- run/rerun submission -->
+                    <!--  run: status = submitting auto_run = false  -->
+                    <!--  rerun: else   -->
+                    <span data-tooltip="{ submission.status === 'Submitting' && !submission.auto_run ? 'Run Submission' : 'Rerun Submission' }"
+                        data-inverted=""
+                        onclick="{ submission.status === 'Submitting' && !submission.auto_run ? run_submission.bind(this, submission) : rerun_submission.bind(this, submission) }">
+                        <i class="icon { submission.status === 'Submitting' && !submission.auto_run ? 'green play' : 'blue redo' }"></i>
                     </span>
+                    <!-- delete submission -->
                     <span data-tooltip="Delete Submission"
                           data-inverted=""
                           onclick="{ delete_submission.bind(this, submission) }">
                         <i class="icon red trash alternate"></i>
-                        <!-- delete submission -->
                     </span>
                 </virtual>
+                <!-- cancel submission -->
                 <span if="{!_.includes(['Finished', 'Cancelled', 'Unknown', 'Failed'], submission.status)}"
                         data-tooltip="Cancel Submission"
                         data-inverted=""
                         onclick="{ cancel_submission.bind(this, submission) }">
                     <i class="grey minus circle icon"></i>
-                    <!-- cancel submission -->
                 </span>
+                <!-- send submission to leaderboard-->
                 <span if="{!submission.on_leaderboard && submission.status === 'Finished'}"
                         data-tooltip="Add to Leaderboard"
                         data-inverted=""
                         onclick="{ add_to_leaderboard.bind(this, submission) }">
                     <i class="icon green columns"></i>
-                    <!-- send submission to leaderboard-->
                 </span>
+                <!--  On leaderboard  -->
                 <span if="{ submission.on_leaderboard }"
                      data-tooltip="On the Leaderboard"
                      data-inverted=""
                      onclick="{ remove_from_leaderboard.bind(this, submission) }">
                     <i class="icon green check"></i>
                 </span>
+                <!--  Make Public  -->
                 <span if="{!submission.is_public && submission.status === 'Finished'}"
                       data-tooltip="Make Public"
                       data-inverted=""
                       onclick="{toggle_submission_is_public.bind(this, submission)}">
                     <i class="icon share teal alternate"></i>
                 </span>
+                <!--  Make Private  -->
                 <span if="{!!submission.is_public && submission.status === 'Finished'}"
                       data-tooltip="Make Private"
                       data-inverted=""
@@ -314,6 +322,23 @@
             }, 100)
         }
 
+        self.run_submission = function (submission) {
+            CODALAB.api.run_submission(submission.id)
+                .done(function (response) {
+                    toastr.success('Submission queued')
+                    self.update_submissions()
+                })
+                .fail(function (response) {
+                    if(response.responseJSON.detail){
+                        toastr.error(response.responseJSON.detail)
+                    } else {
+                        toastr.error(response.responseText)
+                    }
+                })
+            event.stopPropagation()
+
+        }
+
         self.rerun_submission = function (submission) {
             CODALAB.api.re_run_submission(submission.id)
                 .done(function (response) {

src/static/riot/competitions/editor/_competition_details.tag

@@ -136,6 +136,19 @@
                 </a>
             </sup>
         </div>
+        <div class="field">
+            <div class="ui checkbox">
+                <label>Auto-run submissions</label>
+                <input type="checkbox" ref="auto_run_submissions" onchange="{form_updated}">
+            </div>
+            <sup>
+                <span data-tooltip="If unchecked, organizers will have to manually run each submission"
+                          data-inverted=""
+                          data-position="bottom center">
+                    <i class="help icon circle"></i>
+                </span>
+            </sup>
+        </div>
         <div class="field">
             <label>Competition Reward</label>
             <input type="text" ref="reward" placeholder="Example: $1000 for the top participant" onchange="{form_updated}">
@@ -213,6 +226,7 @@
             self.data["description"] = self.markdown_editor.value()
             self.data["queue"] = self.refs.queue.value
             self.data["enable_detailed_results"] = self.refs.detailed_results.checked
+            self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()
@@ -346,6 +360,7 @@
                     .dropdown('set value', competition.queue.id)
             }
             self.refs.detailed_results.checked = competition.enable_detailed_results
+            self.refs.auto_run_submissions.checked = competition.auto_run_submissions
             self.refs.make_programs_available.checked = competition.make_programs_available
             self.refs.make_input_data_available.checked = competition.make_input_data_available
             $(self.refs.docker_image).val(competition.docker_image)