From 7ac9950aa41116df0f6d093aff7d919f3cc42f18 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Fri, 3 Feb 2023 15:31:00 -0800 Subject: [PATCH 1/2] Remove progressbar dependency. --- .../commands/fix_duplicate_assessment_items.py | 5 +---- .../management/commands/set_content_mimetypes.py | 7 +------ .../management/commands/set_storage_used.py | 9 ++++++--- requirements.in | 1 - requirements.txt | 6 ------ 5 files changed, 8 insertions(+), 20 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py b/contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py index 2aa3bf0f28..96c86e3fa5 100644 --- a/contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py +++ b/contentcuration/contentcuration/management/commands/fix_duplicate_assessment_items.py @@ -2,14 +2,12 @@ import time import uuid -import progressbar from django.core.management.base import BaseCommand from django.db.models import Count from django.db.models import F from contentcuration.models import ContentNode -logmodule.basicConfig() logging = logmodule.getLogger(__name__) @@ -27,7 +25,6 @@ def handle(self, *args, **options): logging.info("Fixing {} nodes...".format(total)) - bar = progressbar.ProgressBar(max_value=total) for i, node in enumerate(nodes): # Go through each node's assessment items for item in node.assessment_items.all(): @@ -51,6 +48,6 @@ def handle(self, *args, **options): new_id = uuid.uuid4().hex item.assessment_id = new_id item.save() - bar.update(i) + logging.info("Fixed assessment items for {} node(s)".format(i + 1)) logging.info("Finished in {}".format(time.time() - start)) diff --git a/contentcuration/contentcuration/management/commands/set_content_mimetypes.py b/contentcuration/contentcuration/management/commands/set_content_mimetypes.py index 72c5b57d8c..732d64f8d6 100755 --- a/contentcuration/contentcuration/management/commands/set_content_mimetypes.py +++ b/contentcuration/contentcuration/management/commands/set_content_mimetypes.py @@ -11,7 +11,6 @@ import concurrent.futures import os -import progressbar from django.core.files.storage import default_storage from django.core.management.base import BaseCommand @@ -26,15 +25,11 @@ def handle(self, *args, **kwargs): futures = [] with concurrent.futures.ThreadPoolExecutor() as e: print("Scheduling all metadata update jobs...") - progbar = progressbar.ProgressBar() - for blob in progbar(blobs): + for blob in blobs: future = e.submit(self._update_metadata, blob) futures.append(future) print("Waiting for all jobs to finish...") - progbar = progressbar.ProgressBar(max_value=len(futures)) - for _ in progbar(concurrent.futures.as_completed(futures)): - pass def _determine_cache_control(self, name): _, ext = os.path.splitext(name) diff --git a/contentcuration/contentcuration/management/commands/set_storage_used.py b/contentcuration/contentcuration/management/commands/set_storage_used.py index 33ba1c3ff9..906ac580e7 100644 --- a/contentcuration/contentcuration/management/commands/set_storage_used.py +++ b/contentcuration/contentcuration/management/commands/set_storage_used.py @@ -1,16 +1,19 @@ -import progressbar +import logging + from django.core.management.base import BaseCommand from contentcuration.models import User +logger = logging.getLogger(__name__) + + class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--force", action="store_true", dest="force", default=False) def handle(self, *args, **options): users = User.objects.all() if options["force"] else User.objects.filter(disk_space_used=0) - bar = progressbar.ProgressBar(max_value=users.count()) for index, user in enumerate(users): user.set_space_used() - bar.update(index) + logger.info("Updated storage used for {} user(s)".format(index + 1)) diff --git a/requirements.in b/requirements.in index 6de2c0090c..39f8c21ae3 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,6 @@ newrelic>=2.86.3.70 celery==5.2.7 redis pathlib -progressbar2==3.55.0 python-postmark==0.5.8 Django==3.2.14 django-webpack-loader==0.7.0 diff --git a/requirements.txt b/requirements.txt index 4e998bd8b1..23818cdaa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -179,8 +179,6 @@ pathlib==1.0.1 # via -r requirements.in pillow==9.3.0 # via -r requirements.in -progressbar2==3.55.0 - # via -r requirements.in prometheus-client==0.10.1 # via django-prometheus prompt-toolkit==3.0.23 @@ -219,8 +217,6 @@ python-dateutil==2.8.1 # botocore python-postmark==0.5.8 # via -r requirements.in -python-utils==2.5.6 - # via progressbar2 pytz==2022.1 # via # celery @@ -254,9 +250,7 @@ six==1.16.0 # grpcio # html5lib # oauth2client - # progressbar2 # python-dateutil - # python-utils sqlparse==0.4.1 # via django urllib3==1.26.5 From 053c8b1d1a1a0bca6411d0ce21a92e21f353b92c Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Fri, 3 Feb 2023 15:34:14 -0800 Subject: [PATCH 2/2] Remove progressbar dependency. Remove superflous logging setup that causes duplicate logging. --- .../management/commands/count_public_resources.py | 1 - .../contentcuration/management/commands/fix_exercise_complete.py | 1 - .../contentcuration/management/commands/garbage_collect.py | 1 - .../contentcuration/management/commands/mark_incomplete.py | 1 - .../management/commands/reconcile_change_tasks.py | 1 - .../contentcuration/management/commands/restore_channel.py | 1 - .../management/commands/set_default_learning_activities.py | 1 - .../contentcuration/management/commands/set_file_duration.py | 1 - .../management/commands/set_orm_based_has_captions.py | 1 - contentcuration/contentcuration/management/commands/setup.py | 1 - 10 files changed, 10 deletions(-) diff --git a/contentcuration/contentcuration/management/commands/count_public_resources.py b/contentcuration/contentcuration/management/commands/count_public_resources.py index a9edd48bca..54759a780e 100644 --- a/contentcuration/contentcuration/management/commands/count_public_resources.py +++ b/contentcuration/contentcuration/management/commands/count_public_resources.py @@ -5,7 +5,6 @@ from contentcuration.models import Channel from contentcuration.models import ContentNode -logging.basicConfig() logger = logging.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/fix_exercise_complete.py b/contentcuration/contentcuration/management/commands/fix_exercise_complete.py index 74b3438302..f9ed6e903f 100644 --- a/contentcuration/contentcuration/management/commands/fix_exercise_complete.py +++ b/contentcuration/contentcuration/management/commands/fix_exercise_complete.py @@ -9,7 +9,6 @@ from contentcuration.models import ContentNode from contentcuration.models import License -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/garbage_collect.py b/contentcuration/contentcuration/management/commands/garbage_collect.py index f22f70dd4b..732a494aec 100644 --- a/contentcuration/contentcuration/management/commands/garbage_collect.py +++ b/contentcuration/contentcuration/management/commands/garbage_collect.py @@ -16,7 +16,6 @@ from contentcuration.utils.garbage_collect import clean_up_tasks -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/mark_incomplete.py b/contentcuration/contentcuration/management/commands/mark_incomplete.py index 0e058b4e95..056634d7d8 100644 --- a/contentcuration/contentcuration/management/commands/mark_incomplete.py +++ b/contentcuration/contentcuration/management/commands/mark_incomplete.py @@ -13,7 +13,6 @@ from contentcuration.models import File from contentcuration.models import License -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/reconcile_change_tasks.py b/contentcuration/contentcuration/management/commands/reconcile_change_tasks.py index 6fde77c781..4aa2f9f261 100644 --- a/contentcuration/contentcuration/management/commands/reconcile_change_tasks.py +++ b/contentcuration/contentcuration/management/commands/reconcile_change_tasks.py @@ -6,7 +6,6 @@ from contentcuration.models import Change from contentcuration.models import User -logging.basicConfig() logger = logging.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/restore_channel.py b/contentcuration/contentcuration/management/commands/restore_channel.py index 4088232c4b..efaeb3ee7c 100644 --- a/contentcuration/contentcuration/management/commands/restore_channel.py +++ b/contentcuration/contentcuration/management/commands/restore_channel.py @@ -4,7 +4,6 @@ from contentcuration.utils.import_tools import import_channel -logging.basicConfig() logger = logging.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/set_default_learning_activities.py b/contentcuration/contentcuration/management/commands/set_default_learning_activities.py index 9d958227e0..b6202477fe 100644 --- a/contentcuration/contentcuration/management/commands/set_default_learning_activities.py +++ b/contentcuration/contentcuration/management/commands/set_default_learning_activities.py @@ -6,7 +6,6 @@ from contentcuration.constants.contentnode import kind_activity_map from contentcuration.models import ContentNode -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/set_file_duration.py b/contentcuration/contentcuration/management/commands/set_file_duration.py index 1e828dac05..77446c9853 100644 --- a/contentcuration/contentcuration/management/commands/set_file_duration.py +++ b/contentcuration/contentcuration/management/commands/set_file_duration.py @@ -7,7 +7,6 @@ from contentcuration.models import File from contentcuration.models import MEDIA_PRESETS -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/set_orm_based_has_captions.py b/contentcuration/contentcuration/management/commands/set_orm_based_has_captions.py index edbcbbcd40..29769e4389 100644 --- a/contentcuration/contentcuration/management/commands/set_orm_based_has_captions.py +++ b/contentcuration/contentcuration/management/commands/set_orm_based_has_captions.py @@ -11,7 +11,6 @@ from contentcuration.models import ContentNode from contentcuration.models import File -logmodule.basicConfig(level=logmodule.INFO) logging = logmodule.getLogger('command') diff --git a/contentcuration/contentcuration/management/commands/setup.py b/contentcuration/contentcuration/management/commands/setup.py index 5a41ed9e7b..3284349ebe 100644 --- a/contentcuration/contentcuration/management/commands/setup.py +++ b/contentcuration/contentcuration/management/commands/setup.py @@ -26,7 +26,6 @@ from contentcuration.utils.publish import publish_channel from contentcuration.utils.storage_common import is_gcs_backend -logmodule.basicConfig() logging = logmodule.getLogger(__name__) DESCRIPTION = """