From 7d01068570effb91e4641e8239e8bcb2d9b75e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrien=20Pav=C3=A3o?= Date: Wed, 21 Aug 2024 01:07:18 -0300 Subject: [PATCH 1/5] Changed submission name to add ID when downloading many submissions (#1565) * changed submission name to add ID * Flake8 white space fix --------- Co-authored-by: Nicolas HOMBERG --- src/apps/api/views/submissions.py | 6 +++--- src/apps/competitions/tasks.py | 7 +------ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/apps/api/views/submissions.py b/src/apps/api/views/submissions.py index 745cdb521..6c2489245 100644 --- a/src/apps/api/views/submissions.py +++ b/src/apps/api/views/submissions.py @@ -312,18 +312,18 @@ def re_run_many_submissions(self, request): submission.re_run() return Response({}) - # New methods impleted! @action(detail=False, methods=['get']) def download_many(self, request): pks = request.query_params.get('pks') if pks: pks = json.loads(pks) # Convert JSON string to list + # Doing a local import here to avoid circular imports from competitions.tasks import stream_batch_download - # Call the task and get the result (stream) + # in_memory_zip = stream_batch_download.apply_async((pks,)).get() in_memory_zip = stream_batch_download(pks) - # Stream the response + response = StreamingHttpResponse(in_memory_zip, content_type='application/zip') response['Content-Disposition'] = 'attachment; filename="bulk_submissions.zip"' return response diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 1e5544b7b..f56696940 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -292,13 +292,10 @@ def retrieve_data(url, data=None): def zip_generator(submission_pks): in_memory_zip = BytesIO() - # logger.info("IN zip generator") with zipfile.ZipFile(in_memory_zip, 'w', zipfile.ZIP_DEFLATED) as zip_file: for submission_id in submission_pks: submission = Submission.objects.get(id=submission_id) - # logger.info(submission.data.data_file) - - short_name = submission.data.data_file.name.split('/')[-1] + short_name = "ID_" + str(submission_id) + '_' + submission.data.data_file.name.split('/')[-1] url = make_url_sassy(path=submission.data.data_file.name) for block in retrieve_data(url): zip_file.writestr(short_name, block) @@ -310,8 +307,6 @@ def zip_generator(submission_pks): @app.task(queue='site-worker', soft_time_limit=60 * 60) def stream_batch_download(submission_pks): - # logger.info("In stream_batch_download") - # logger.info(submission_pks) return zip_generator(submission_pks) From a37139a96d7f2beb169b40e9e322453545be5673 Mon Sep 17 00:00:00 2001 From: Ihsan Ullah Date: Wed, 21 Aug 2024 19:01:39 +0500 Subject: [PATCH 2/5] Server Status: submissions limit removed (#1562) * server status submissions limit removed, pagination added, monitor queues separated * renamed server status title to recent submissions --- src/apps/pages/urls.py | 1 + src/apps/pages/views.py | 29 +++++++++--- src/static/stylus/index.styl | 1 + src/static/stylus/server_status.styl | 19 ++++++++ src/templates/base.html | 1 + src/templates/pages/monitor_queues.html | 49 ++++++++++++++++++++ src/templates/pages/server_status.html | 61 +++++++++---------------- 7 files changed, 116 insertions(+), 45 deletions(-) create mode 100644 src/static/stylus/server_status.styl create mode 100644 src/templates/pages/monitor_queues.html diff --git a/src/apps/pages/urls.py b/src/apps/pages/urls.py index 084bd2c7e..bbba17d71 100644 --- a/src/apps/pages/urls.py +++ b/src/apps/pages/urls.py @@ -11,5 +11,6 @@ path('search', views.SearchView.as_view(), name="search"), path('organize', views.OrganizeView.as_view(), name="organize"), path('server_status', views.ServerStatusView.as_view(), name="server_status"), + path('monitor_queues', views.MonitorQueuesView.as_view(), name="monitor_queues"), # path('test', views.CompetitionListTestView.as_view()), ] diff --git a/src/apps/pages/views.py b/src/apps/pages/views.py index 4afa0c11d..0f21b8cf2 100644 --- a/src/apps/pages/views.py +++ b/src/apps/pages/views.py @@ -1,5 +1,4 @@ -from datetime import timedelta -from django.utils.timezone import now +from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage from django.views.generic import TemplateView from django.db.models import Count, Q @@ -55,6 +54,8 @@ class ServerStatusView(TemplateView): def get_context_data(self, *args, **kwargs): show_child_submissions = self.request.GET.get('show_child_submissions', False) + page = self.request.GET.get('page', 1) + submissions_per_page = 50 # Get all submissions qs = Submission.objects.all() @@ -73,9 +74,6 @@ def get_context_data(self, *args, **kwargs): else: qs = qs.none() # This returns an empty queryset - # Filter for fetching last 2 days submissions - qs = qs.filter(created_when__gte=now() - timedelta(days=2)) - # Filter out child submissions i.e. submission has no parent if not show_child_submissions: qs = qs.filter(parent__isnull=True) @@ -83,8 +81,20 @@ def get_context_data(self, *args, **kwargs): qs = qs.order_by('-created_when') qs = qs.select_related('phase__competition', 'owner') + # Paginate the queryset + paginator = Paginator(qs, submissions_per_page) + + try: + submissions = paginator.page(page) + except PageNotAnInteger: + # If page is not an integer, deliver the first page. + submissions = paginator.page(1) + except EmptyPage: + # If page is out of range, deliver last page of results. + submissions = paginator.page(paginator.num_pages) + context = super().get_context_data(*args, **kwargs) - context['submissions'] = qs[:250] + context['submissions'] = submissions context['show_child_submissions'] = show_child_submissions for submission in context['submissions']: @@ -103,6 +113,9 @@ def get_context_data(self, *args, **kwargs): # Add submission owner display name submission.owner_display_name = submission.owner.display_name if submission.owner.display_name else submission.owner.username + context['paginator'] = paginator + context['is_paginated'] = paginator.num_pages > 1 + return context def format_file_size(self, file_size): @@ -123,6 +136,10 @@ def format_file_size(self, file_size): return f"{n:.1f} {units[i]}" +class MonitorQueuesView(TemplateView): + template_name = 'pages/monitor_queues.html' + + def page_not_found_view(request, exception): print(request) return render(request, '404.html', status=404) diff --git a/src/static/stylus/index.styl b/src/static/stylus/index.styl index f68e56b2b..58cfc96bc 100644 --- a/src/static/stylus/index.styl +++ b/src/static/stylus/index.styl @@ -7,4 +7,5 @@ @import "src/static/stylus/simple_page.styl" @import "src/static/stylus/toastr.styl" @import "src/static/stylus/forum.styl" +@import "src/static/stylus/server_status.styl" diff --git a/src/static/stylus/server_status.styl b/src/static/stylus/server_status.styl new file mode 100644 index 000000000..2f96a3fe9 --- /dev/null +++ b/src/static/stylus/server_status.styl @@ -0,0 +1,19 @@ +.pagination-nav + padding 10px 0 + width 100% + margin-bottom 20px + display flex + justify-content center + align-items center + position relative + +.float-left + position absolute + left 0 + +.float-right + position absolute + right 0 + +.center + text-align center diff --git a/src/templates/base.html b/src/templates/base.html index de6296321..f6f650d5a 100644 --- a/src/templates/base.html +++ b/src/templates/base.html @@ -128,6 +128,7 @@ {# Customize Codalab#} Server Status {% if request.user.is_staff %} + Monitor Queues Django Admin Change User Analytics diff --git a/src/templates/pages/monitor_queues.html b/src/templates/pages/monitor_queues.html new file mode 100644 index 000000000..bcbf0c4c5 --- /dev/null +++ b/src/templates/pages/monitor_queues.html @@ -0,0 +1,49 @@ +{% extends "base.html" %} +{% load staticfiles %} + +{% block extra_head %} +{% endblock %} + +{% block content %} +
+

Monitor queues

+
+
+
+ + + +
+ RabbitMQ +
+ This page allows admins to view connections, queued messages, message rates, channels, + exchanges, and other administrative features relating to RabbitMQ e.g. Creating users, + adding v-hosts, and creating policies. +
+
+
+
+
+
+ + + +
+ Flower +
+ Flower is a powerful web-based Celery monitoring tool designed to keep track of our + tasks. + Admins may view the state of which tasks were run, with what arguments, and many more + features. Here you may also view which queues your celery workers are consuming, and the + state of any tasks in them. At last, there is also a great monitoring page for viewing + the + systemic impact of your workers. +
+
+
+
+
+
+ + +{% endblock %} diff --git a/src/templates/pages/server_status.html b/src/templates/pages/server_status.html index debf5ba1a..dde0138f1 100644 --- a/src/templates/pages/server_status.html +++ b/src/templates/pages/server_status.html @@ -6,7 +6,7 @@ {% block content %}
-

Recent submissions (up to 250 or 2 days old)

+

Recent Submissions

@@ -55,48 +55,31 @@

Recent submissions (up to 250 or 2 days old)

+ + {% if is_paginated %} +
+ {% if submissions.has_previous %} + + First + Previous + + {% endif %} + + + Page {{ submissions.number }} of {{ paginator.num_pages }} + + + + {% if submissions.has_next %} + + Next + Last + + {% endif %} - {% if user.is_superuser %} -

Monitor queues

-
-
-
- - - -
- RabbitMQ -
- This page allows admins to view connections, queued messages, message rates, channels, - exchanges, and other administrative features relating to RabbitMQ e.g. Creating users, - adding v-hosts, and creating policies. -
-
-
-
-
-
- - - -
- Flower -
- Flower is a powerful web-based Celery monitoring tool designed to keep track of our - tasks. - Admins may view the state of which tasks were run, with what arguments, and many more - features. Here you may also view which queues your celery workers are consuming, and the - state of any tasks in them. At last, there is also a great monitoring page for viewing - the - systemic impact of your workers. -
-
-
-
{% endif %}
-