From 0c908616372ab9346dc0eb6b027f0cc41d64e2f9 Mon Sep 17 00:00:00 2001 From: Henrry Pulgarin <39854568+Henrrypg@users.noreply.github.com> Date: Tue, 18 Oct 2022 13:53:11 -0500 Subject: [PATCH 01/22] feat: add survey_report djangoapp --- lms/envs/common.py | 3 ++ openedx/features/survey_report/__init__.py | 0 openedx/features/survey_report/apps.py | 12 +++++++ .../survey_report/migrations/0001_initial.py | 34 +++++++++++++++++++ .../survey_report/migrations/__init__.py | 0 openedx/features/survey_report/models.py | 32 +++++++++++++++++ 6 files changed, 81 insertions(+) create mode 100644 openedx/features/survey_report/__init__.py create mode 100644 openedx/features/survey_report/apps.py create mode 100644 openedx/features/survey_report/migrations/0001_initial.py create mode 100644 openedx/features/survey_report/migrations/__init__.py create mode 100644 openedx/features/survey_report/models.py diff --git a/lms/envs/common.py b/lms/envs/common.py index 407fea2b1e2b..bbdc725d07a4 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -3239,6 +3239,9 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # Agreements 'openedx.core.djangoapps.agreements', + # Survey reports + 'openedx.features.survey_report', + # User and group management via edx-django-utils 'edx_django_utils.user', diff --git a/openedx/features/survey_report/__init__.py b/openedx/features/survey_report/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/survey_report/apps.py b/openedx/features/survey_report/apps.py new file mode 100644 index 000000000000..34786a48773c --- /dev/null +++ b/openedx/features/survey_report/apps.py @@ -0,0 +1,12 @@ +""" +Survey Report App Configuration. +""" +from django.apps import AppConfig + + +class SurveyReportConfig(AppConfig): + """ + Configuration for the survey report Django app. + """ + default_auto_field = 'django.db.models.BigAutoField' + name = 'openedx.features.survey_report' diff --git a/openedx/features/survey_report/migrations/0001_initial.py b/openedx/features/survey_report/migrations/0001_initial.py new file mode 100644 index 000000000000..a7595569917e --- /dev/null +++ b/openedx/features/survey_report/migrations/0001_initial.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.16 on 2022-10-21 14:54 + +from django.db import migrations, models +import jsonfield.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='SurveyReport', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('courses_offered', models.IntegerField()), + ('learners', models.IntegerField()), + ('registered_learners', models.IntegerField()), + ('enrollments', models.IntegerField()), + ('generated_certificates', models.IntegerField()), + ('extra_data', jsonfield.fields.JSONField(blank=True, default={}, help_text='Extra information for instance data')), + ('request_details', models.CharField(blank=True, max_length=255, null=True)), + ('created_at', models.DateTimeField(auto_now=True)), + ('sent_at', models.DateTimeField(blank=True, null=True)), + ], + options={ + 'ordering': ['-created_at'], + 'get_latest_by': 'created_at', + }, + ), + ] diff --git a/openedx/features/survey_report/migrations/__init__.py b/openedx/features/survey_report/migrations/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py new file mode 100644 index 000000000000..a9b3c2dd017b --- /dev/null +++ b/openedx/features/survey_report/models.py @@ -0,0 +1,32 @@ +""" +Survey Report models. +""" + +from jsonfield import JSONField + +from django.db import models + + +class SurveyReport(models.Model): + """ + This model stores information to automate the way of gathering impact data from the openedx project. + + .. no_pii: + """ + courses_offered = models.IntegerField() + learners = models.IntegerField() + registered_learners = models.IntegerField() + enrollments = models.IntegerField() + generated_certificates = models.IntegerField() + extra_data = JSONField( + blank=True, + default={}, + help_text="Extra information for instance data", + ) + request_details = models.CharField(max_length=255, null=True, blank=True) + created_at = models.DateTimeField(auto_now=True) + sent_at = models.DateTimeField(null=True, blank=True) + + class Meta: + ordering = ["-created_at"] + get_latest_by = 'created_at' From 1a85171b86f8a4eae86bf4a739633cfe50e1dec2 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Wed, 26 Oct 2022 08:53:11 -0500 Subject: [PATCH 02/22] feat: add survey report cli command and query methods Co-authored-by: henrrypg --- lms/envs/production.py | 3 + lms/envs/test.py | 3 + openedx/features/survey_report/application.py | 35 ++++++ .../survey_report/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/generate_report.py | 28 +++++ .../management/commands/tests/__init__.py | 0 .../commands/tests/test_generate_report.py | 44 +++++++ openedx/features/survey_report/models.py | 3 +- openedx/features/survey_report/queries.py | 58 +++++++++ .../survey_report/tests/test_query_methods.py | 114 ++++++++++++++++++ 11 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 openedx/features/survey_report/application.py create mode 100644 openedx/features/survey_report/management/__init__.py create mode 100644 openedx/features/survey_report/management/commands/__init__.py create mode 100644 openedx/features/survey_report/management/commands/generate_report.py create mode 100644 openedx/features/survey_report/management/commands/tests/__init__.py create mode 100644 openedx/features/survey_report/management/commands/tests/test_generate_report.py create mode 100644 openedx/features/survey_report/queries.py create mode 100644 openedx/features/survey_report/tests/test_query_methods.py diff --git a/lms/envs/production.py b/lms/envs/production.py index d1c5a08aa2b0..8455f76948a0 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -1080,3 +1080,6 @@ def get_env_setting(setting): "SECRET": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_SECRET', None), "URL": ENV_TOKENS.get('BIG_BLUE_BUTTON_GLOBAL_URL', None), } + +############## Settings for survey report ############## +SURVEY_REPORT_EXTRA_DATA = ENV_TOKENS.get('SURVEY_REPORT_EXTRA_DATA', {}) diff --git a/lms/envs/test.py b/lms/envs/test.py index 94bbb6170903..634c2bb627d4 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -671,3 +671,6 @@ "LOGO_URL": "https://courses.example.com/yourmfe-logo.png", }, } + +############## Settings for survey report ############## +SURVEY_REPORT_EXTRA_DATA = {} diff --git a/openedx/features/survey_report/application.py b/openedx/features/survey_report/application.py new file mode 100644 index 000000000000..776433d8ce98 --- /dev/null +++ b/openedx/features/survey_report/application.py @@ -0,0 +1,35 @@ +""" +Contains the logic to manage survey report model. +""" + +from django.conf import settings + +from openedx.features.survey_report.models import SurveyReport +from openedx.features.survey_report.queries import ( + get_course_enrollments, + get_currently_learners, + get_generated_certificates, + get_learners_registered, + get_unique_courses_offered +) + + +def generate_report() -> None: + """ Generate a report with relevant data.""" + courses_offered = get_unique_courses_offered() + learners = get_currently_learners() + registered = get_learners_registered() + certificates = get_generated_certificates() + enrollments = get_course_enrollments() + extra_data = settings.SURVEY_REPORT_EXTRA_DATA + + survey_report = SurveyReport( + courses_offered=courses_offered, + learners=learners, + registered_learners=registered, + generated_certificates=certificates, + enrollments=enrollments, + extra_data=extra_data, + ) + + survey_report.save() diff --git a/openedx/features/survey_report/management/__init__.py b/openedx/features/survey_report/management/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/survey_report/management/commands/__init__.py b/openedx/features/survey_report/management/commands/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/survey_report/management/commands/generate_report.py b/openedx/features/survey_report/management/commands/generate_report.py new file mode 100644 index 000000000000..475469dbfe37 --- /dev/null +++ b/openedx/features/survey_report/management/commands/generate_report.py @@ -0,0 +1,28 @@ +""" +CLI command to generate survey report. +""" + +from django.core.management.base import BaseCommand, CommandError + +from openedx.features.survey_report.application import generate_report + + +class Command(BaseCommand): + """ + Management command to generate a new survey report with + non-sensitive data. + """ + + help = """ + This command will create a new survey report using some + models to get the total number oof courses offered, currently active learners, + learners ever registered, and generated certificates. + """ + + def handle(self, *_args, **_options): + try: + generate_report() + except Exception as error: + raise CommandError(f'An error has occurred while report was generating. {error}') from error + + self.stdout.write(self.style.SUCCESS('Survey report has been generated successfully.')) diff --git a/openedx/features/survey_report/management/commands/tests/__init__.py b/openedx/features/survey_report/management/commands/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/features/survey_report/management/commands/tests/test_generate_report.py b/openedx/features/survey_report/management/commands/tests/test_generate_report.py new file mode 100644 index 000000000000..45f0d553c86d --- /dev/null +++ b/openedx/features/survey_report/management/commands/tests/test_generate_report.py @@ -0,0 +1,44 @@ +""" +Test for generate_report command. +""" + +from io import StringIO +from unittest import mock + +from django.core.management import call_command +from django.test import TestCase, override_settings + +from openedx.features.survey_report.models import SurveyReport + + +class GenerateReportTest(TestCase): + """ + Test for generate_report command. + """ + @override_settings(SURVEY_REPORT_EXTRA_DATA={'extra_data': 'extra_data'}) + @mock.patch('openedx.features.survey_report.queries.get_course_enrollments') + @mock.patch('openedx.features.survey_report.queries.get_generated_certificates') + @mock.patch('openedx.features.survey_report.queries.get_learners_registered') + @mock.patch('openedx.features.survey_report.queries.get_currently_learners') + @mock.patch('openedx.features.survey_report.queries.get_unique_courses_offered') + def test_generate_report(self, mock_get_unique_courses_offered, mock_get_currently_learners, + mock_get_learners_registered, mock_get_generated_certificates, + mock_get_course_enrollments): + """ + Test that generate_report command creates a survey report. + """ + mock_get_unique_courses_offered.return_value = 1 + mock_get_currently_learners.return_value = 2 + mock_get_learners_registered.return_value = 3 + mock_get_generated_certificates.return_value = 4 + mock_get_course_enrollments.return_value = 5 + out = StringIO() + call_command('generate_report', stdout=out) + + survey_report = SurveyReport.objects.last() + assert survey_report.courses_offered == 1 + assert survey_report.learners == 2 + assert survey_report.registered_learners == 3 + assert survey_report.generated_certificates == 4 + assert survey_report.enrollments == 5 + assert survey_report.extra_data == {'extra_data': 'extra_data'} diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py index a9b3c2dd017b..d95411438650 100644 --- a/openedx/features/survey_report/models.py +++ b/openedx/features/survey_report/models.py @@ -2,9 +2,8 @@ Survey Report models. """ -from jsonfield import JSONField - from django.db import models +from jsonfield import JSONField class SurveyReport(models.Model): diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py new file mode 100644 index 000000000000..d2e179482cb8 --- /dev/null +++ b/openedx/features/survey_report/queries.py @@ -0,0 +1,58 @@ +""" +Queries to get data from database. +""" + +from datetime import datetime, timedelta + +from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user +from django.db.models import Count, OuterRef, Q, Subquery + +from common.djangoapps.student.models import CourseEnrollment +from lms.djangoapps.grades.models import PersistentCourseGrade +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview + + +def get_unique_courses_offered() -> int: + """ + Get total number of unique courses offered. + """ + return CourseOverview.objects.annotate( + count=Subquery( + CourseEnrollment.objects + .filter(course_id=OuterRef('id')) + .values('course_id') + .annotate(count=Count('course_id')) + .values('count') + ))\ + .filter(count__gt=5)\ + .filter(start__lt=datetime.now())\ + .filter(Q(end__isnull=True) | Q(end__gt=datetime.now()))\ + .count() + + +def get_currently_learners() -> int: + """ + Get total number of learners with last login in the last 3 weeks. + """ + return User.objects.filter(last_login__gte=datetime.now() - timedelta(weeks=3)).count() + + +def get_learners_registered() -> int: + """ + Get total number of active learners registered. + """ + return User.objects.filter(is_active=True).count() + + +def get_generated_certificates() -> int: + """ + Get total number of generated certificates. + """ + return PersistentCourseGrade.objects.filter(passed_timestamp__isnull=False).count() + + +def get_course_enrollments() -> int: + """ + Get total number of enrollments from users that aren't staff. + """ + return CourseEnrollment.objects.filter(user__is_superuser=False, user__is_staff=False).count() diff --git a/openedx/features/survey_report/tests/test_query_methods.py b/openedx/features/survey_report/tests/test_query_methods.py new file mode 100644 index 000000000000..e426a38886f9 --- /dev/null +++ b/openedx/features/survey_report/tests/test_query_methods.py @@ -0,0 +1,114 @@ +""" +Test for survey report commands. +""" + +from datetime import datetime, timedelta +from unittest.mock import patch + +from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory +from lms.djangoapps.grades.models import PersistentCourseGrade +from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory +from openedx.features.survey_report.queries import ( + get_course_enrollments, + get_currently_learners, + get_generated_certificates, + get_learners_registered, + get_unique_courses_offered +) +from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order + + +class TestSurveyReportCommands(ModuleStoreTestCase): + """ + Test for survey report query methods. + """ + + def setUp(self): + """ + Setup for users and courses. + """ + super().setUp() + self.store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) # lint-amnesty, pylint: disable=protected-access + self.first_course = CourseFactory.create( + org="test", course="course1", display_name="run1", default_store=ModuleStoreEnum.Type.mongo + ) + self.user = UserFactory.create(username='test_user', email='test@example.com', password='password') + self.user1 = UserFactory.create(username='test_user1', email='test1@example.com', password='password') + self.user2 = UserFactory.create(username='test_user2', email='test2@example.com', password='password') + self.user3 = UserFactory.create(username='test_user3', email='test3@example.com', password='password') + self.user4 = UserFactory.create(username='test_user4', email='test4@example.com', password='password') + self.user5 = UserFactory.create(username='test_user5', email='test5@example.com', password='password') + + def test_get_unique_courses_offered(self): + """ + Test that get_unique_courses_offered returns the correct number of courses. + """ + course_overview = CourseOverviewFactory.create(id=self.first_course.id, start="2019-01-01", end="2024-01-01") + CourseEnrollmentFactory.create(user=self.user, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user1, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user2, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user3, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user4, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user5, course_id=course_overview.id) + with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: + mock_datetime.now.return_value = datetime.now() + assert get_unique_courses_offered() == 1 + + def test_get_currently_learners(self): + """ + Test that get_currently_learners returns the correct number of learners. + """ + self.user.last_login = datetime.now() - timedelta(days=1) + self.user.save() + self.user1.last_login = datetime.now() - timedelta(weeks=2) + self.user1.save() + self.user2.last_login = datetime.now() - timedelta(weeks=4) + self.user2.save() + with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: + mock_datetime.now.return_value = datetime.now() + assert get_currently_learners() == 2 + + def test_get_learners_registered(self): + """ + Test that get_learners_registered returns the correct number of learners. + """ + assert get_learners_registered() == 7 + + def test_get_generated_certificates(self): + """ + Test that get_generated_certificates returns the correct number of certificates. + """ + course_grade_params = { + "user_id": self.user.id, + "course_id": self.first_course.id, + "percent_grade": 77.7, + "letter_grade": "pass", + "passed": True, + "passed_timestamp": datetime.now(), + } + PersistentCourseGrade.update_or_create(**course_grade_params) + + with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: + mock_datetime.now.return_value = datetime.now() + assert get_generated_certificates() == 1 + + def test_get_course_enrollments(self): + """ + Test that get_course_enrollments returns the correct number of enrollments. + """ + self.user.is_superuser = True + self.user.save() + self.user1.is_staff = True + self.user1.save() + course_overview = CourseOverviewFactory.create(id=self.first_course.id, start="2019-01-01", end="2024-01-01") + CourseEnrollmentFactory.create(user=self.user, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user1, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user2, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user3, course_id=course_overview.id) + CourseEnrollmentFactory.create(user=self.user4, course_id=course_overview.id) + with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: + mock_datetime.now.return_value = datetime.now() + assert get_course_enrollments() == 3 From c06ad16d544e42ac1fc14dbff231687c1bd1a5df Mon Sep 17 00:00:00 2001 From: henrrypg Date: Wed, 26 Oct 2022 09:11:44 -0500 Subject: [PATCH 03/22] fix: add init file --- openedx/features/survey_report/tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 openedx/features/survey_report/tests/__init__.py diff --git a/openedx/features/survey_report/tests/__init__.py b/openedx/features/survey_report/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From 2a11601dcff14dd11c0a92ff36e03ed17000e42f Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Mon, 31 Oct 2022 11:08:16 -0500 Subject: [PATCH 04/22] refactor: change model fields --- .../survey_report/migrations/0001_initial.py | 14 +++++++------- openedx/features/survey_report/models.py | 12 ++++++------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/openedx/features/survey_report/migrations/0001_initial.py b/openedx/features/survey_report/migrations/0001_initial.py index a7595569917e..2ece7f197efc 100644 --- a/openedx/features/survey_report/migrations/0001_initial.py +++ b/openedx/features/survey_report/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.16 on 2022-10-21 14:54 +# Generated by Django 3.2.16 on 2022-10-31 16:52 from django.db import migrations, models import jsonfield.fields @@ -16,12 +16,12 @@ class Migration(migrations.Migration): name='SurveyReport', fields=[ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('courses_offered', models.IntegerField()), - ('learners', models.IntegerField()), - ('registered_learners', models.IntegerField()), - ('enrollments', models.IntegerField()), - ('generated_certificates', models.IntegerField()), - ('extra_data', jsonfield.fields.JSONField(blank=True, default={}, help_text='Extra information for instance data')), + ('courses_offered', models.BigIntegerField()), + ('learners', models.BigIntegerField()), + ('registered_learners', models.BigIntegerField()), + ('enrollments', models.BigIntegerField()), + ('generated_certificates', models.BigIntegerField()), + ('extra_data', jsonfield.fields.JSONField(blank=True, default=dict, help_text='Extra information for instance data')), ('request_details', models.CharField(blank=True, max_length=255, null=True)), ('created_at', models.DateTimeField(auto_now=True)), ('sent_at', models.DateTimeField(blank=True, null=True)), diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py index d95411438650..32ba7f131c03 100644 --- a/openedx/features/survey_report/models.py +++ b/openedx/features/survey_report/models.py @@ -12,14 +12,14 @@ class SurveyReport(models.Model): .. no_pii: """ - courses_offered = models.IntegerField() - learners = models.IntegerField() - registered_learners = models.IntegerField() - enrollments = models.IntegerField() - generated_certificates = models.IntegerField() + courses_offered = models.BigIntegerField() + learners = models.BigIntegerField() + registered_learners = models.BigIntegerField() + enrollments = models.BigIntegerField() + generated_certificates = models.BigIntegerField() extra_data = JSONField( blank=True, - default={}, + default=dict, help_text="Extra information for instance data", ) request_details = models.CharField(max_length=255, null=True, blank=True) From 4ff635236fc753aefde3acb1d0752edb7f1d3a7c Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Mon, 31 Oct 2022 16:11:14 -0500 Subject: [PATCH 05/22] refactor: rename application file and rename methods --- .../survey_report/{application.py => api.py} | 8 +++--- .../management/commands/generate_report.py | 2 +- openedx/features/survey_report/queries.py | 27 ++++++++++++++----- .../survey_report/tests/test_query_methods.py | 8 +++--- 4 files changed, 29 insertions(+), 16 deletions(-) rename openedx/features/survey_report/{application.py => api.py} (82%) diff --git a/openedx/features/survey_report/application.py b/openedx/features/survey_report/api.py similarity index 82% rename from openedx/features/survey_report/application.py rename to openedx/features/survey_report/api.py index 776433d8ce98..828eabcc6466 100644 --- a/openedx/features/survey_report/application.py +++ b/openedx/features/survey_report/api.py @@ -7,9 +7,9 @@ from openedx.features.survey_report.models import SurveyReport from openedx.features.survey_report.queries import ( get_course_enrollments, - get_currently_learners, + get_active_learners_in_the_last_weeks, get_generated_certificates, - get_learners_registered, + get_registered_learners, get_unique_courses_offered ) @@ -17,8 +17,8 @@ def generate_report() -> None: """ Generate a report with relevant data.""" courses_offered = get_unique_courses_offered() - learners = get_currently_learners() - registered = get_learners_registered() + learners = get_active_learners_in_the_last_weeks(weeks=3) + registered = get_registered_learners() certificates = get_generated_certificates() enrollments = get_course_enrollments() extra_data = settings.SURVEY_REPORT_EXTRA_DATA diff --git a/openedx/features/survey_report/management/commands/generate_report.py b/openedx/features/survey_report/management/commands/generate_report.py index 475469dbfe37..e08d89ae4924 100644 --- a/openedx/features/survey_report/management/commands/generate_report.py +++ b/openedx/features/survey_report/management/commands/generate_report.py @@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand, CommandError -from openedx.features.survey_report.application import generate_report +from openedx.features.survey_report.api import generate_report class Command(BaseCommand): diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index d2e179482cb8..fe20586ea0a0 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -7,6 +7,7 @@ from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user from django.db.models import Count, OuterRef, Q, Subquery +from common.djangoapps.util.query import read_replica_or_default from common.djangoapps.student.models import CourseEnrollment from lms.djangoapps.grades.models import PersistentCourseGrade from openedx.core.djangoapps.content.course_overviews.models import CourseOverview @@ -27,32 +28,44 @@ def get_unique_courses_offered() -> int: .filter(count__gt=5)\ .filter(start__lt=datetime.now())\ .filter(Q(end__isnull=True) | Q(end__gt=datetime.now()))\ + .using(read_replica_or_default())\ .count() -def get_currently_learners() -> int: +def get_active_learners_in_the_last_weeks(weeks: int) -> int: """ - Get total number of learners with last login in the last 3 weeks. + Get total number of users with last login in the last weeks. + + Args: + weeks (int): number of weeks since the last login to considerate as an active learner. """ - return User.objects.filter(last_login__gte=datetime.now() - timedelta(weeks=3)).count() + return User.objects.filter(last_login__gte=datetime.now() - timedelta(weeks=weeks))\ + .using(read_replica_or_default())\ + .count() -def get_learners_registered() -> int: +def get_registered_learners() -> int: """ Get total number of active learners registered. """ - return User.objects.filter(is_active=True).count() + return User.objects.filter(is_active=True)\ + .using(read_replica_or_default())\ + .count() def get_generated_certificates() -> int: """ Get total number of generated certificates. """ - return PersistentCourseGrade.objects.filter(passed_timestamp__isnull=False).count() + return PersistentCourseGrade.objects.filter(passed_timestamp__isnull=False)\ + .using(read_replica_or_default())\ + .count() def get_course_enrollments() -> int: """ Get total number of enrollments from users that aren't staff. """ - return CourseEnrollment.objects.filter(user__is_superuser=False, user__is_staff=False).count() + return CourseEnrollment.objects.filter(user__is_superuser=False, user__is_staff=False)\ + .using(read_replica_or_default())\ + .count() diff --git a/openedx/features/survey_report/tests/test_query_methods.py b/openedx/features/survey_report/tests/test_query_methods.py index e426a38886f9..9563bbc0ad59 100644 --- a/openedx/features/survey_report/tests/test_query_methods.py +++ b/openedx/features/survey_report/tests/test_query_methods.py @@ -10,9 +10,9 @@ from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from openedx.features.survey_report.queries import ( get_course_enrollments, - get_currently_learners, + get_active_learners_in_the_last_weeks, get_generated_certificates, - get_learners_registered, + get_registered_learners, get_unique_courses_offered ) from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order @@ -69,13 +69,13 @@ def test_get_currently_learners(self): self.user2.save() with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: mock_datetime.now.return_value = datetime.now() - assert get_currently_learners() == 2 + assert get_active_learners_in_the_last_weeks(weeks=3) == 2 def test_get_learners_registered(self): """ Test that get_learners_registered returns the correct number of learners. """ - assert get_learners_registered() == 7 + assert get_registered_learners() == 7 def test_get_generated_certificates(self): """ From 5387bce124c86b00b6f3327449143c77c64f426d Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Mon, 31 Oct 2022 16:19:29 -0500 Subject: [PATCH 06/22] refactor: add is_active to get course enrollments --- openedx/features/survey_report/queries.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index fe20586ea0a0..7f6b52b41f9c 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -64,8 +64,8 @@ def get_generated_certificates() -> int: def get_course_enrollments() -> int: """ - Get total number of enrollments from users that aren't staff. + Get total number of enrollments from users that aren't staff. Course staff members will be included. """ - return CourseEnrollment.objects.filter(user__is_superuser=False, user__is_staff=False)\ + return CourseEnrollment.objects.filter(is_active=True, user__is_superuser=False, user__is_staff=False)\ .using(read_replica_or_default())\ .count() From 2930cbe5d6917bacec94aa566c6060e47d9da779 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 3 Nov 2022 07:12:45 -0500 Subject: [PATCH 07/22] refactor: rename method to get active users --- openedx/features/survey_report/api.py | 4 ++-- openedx/features/survey_report/queries.py | 2 +- openedx/features/survey_report/tests/test_query_methods.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openedx/features/survey_report/api.py b/openedx/features/survey_report/api.py index 828eabcc6466..327a99febff0 100644 --- a/openedx/features/survey_report/api.py +++ b/openedx/features/survey_report/api.py @@ -7,7 +7,7 @@ from openedx.features.survey_report.models import SurveyReport from openedx.features.survey_report.queries import ( get_course_enrollments, - get_active_learners_in_the_last_weeks, + get_recently_active_users, get_generated_certificates, get_registered_learners, get_unique_courses_offered @@ -17,7 +17,7 @@ def generate_report() -> None: """ Generate a report with relevant data.""" courses_offered = get_unique_courses_offered() - learners = get_active_learners_in_the_last_weeks(weeks=3) + learners = get_recently_active_users(weeks=3) registered = get_registered_learners() certificates = get_generated_certificates() enrollments = get_course_enrollments() diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index 7f6b52b41f9c..e1dffd83be02 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -32,7 +32,7 @@ def get_unique_courses_offered() -> int: .count() -def get_active_learners_in_the_last_weeks(weeks: int) -> int: +def get_recently_active_users(weeks: int) -> int: """ Get total number of users with last login in the last weeks. diff --git a/openedx/features/survey_report/tests/test_query_methods.py b/openedx/features/survey_report/tests/test_query_methods.py index 9563bbc0ad59..e7dcadc0626d 100644 --- a/openedx/features/survey_report/tests/test_query_methods.py +++ b/openedx/features/survey_report/tests/test_query_methods.py @@ -10,7 +10,7 @@ from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from openedx.features.survey_report.queries import ( get_course_enrollments, - get_active_learners_in_the_last_weeks, + get_recently_active_users, get_generated_certificates, get_registered_learners, get_unique_courses_offered @@ -69,7 +69,7 @@ def test_get_currently_learners(self): self.user2.save() with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: mock_datetime.now.return_value = datetime.now() - assert get_active_learners_in_the_last_weeks(weeks=3) == 2 + assert get_recently_active_users(weeks=3) == 2 def test_get_learners_registered(self): """ From 9fc6cacf7978068cd4eaba887b51501b653f3b61 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 3 Nov 2022 15:11:47 -0500 Subject: [PATCH 08/22] refactor: remove fields useless --- openedx/features/survey_report/migrations/0001_initial.py | 4 +--- openedx/features/survey_report/models.py | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/openedx/features/survey_report/migrations/0001_initial.py b/openedx/features/survey_report/migrations/0001_initial.py index 2ece7f197efc..46d816a363cc 100644 --- a/openedx/features/survey_report/migrations/0001_initial.py +++ b/openedx/features/survey_report/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.16 on 2022-10-31 16:52 +# Generated by Django 3.2.16 on 2022-11-03 20:07 from django.db import migrations, models import jsonfield.fields @@ -22,9 +22,7 @@ class Migration(migrations.Migration): ('enrollments', models.BigIntegerField()), ('generated_certificates', models.BigIntegerField()), ('extra_data', jsonfield.fields.JSONField(blank=True, default=dict, help_text='Extra information for instance data')), - ('request_details', models.CharField(blank=True, max_length=255, null=True)), ('created_at', models.DateTimeField(auto_now=True)), - ('sent_at', models.DateTimeField(blank=True, null=True)), ], options={ 'ordering': ['-created_at'], diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py index 32ba7f131c03..08faf19902e6 100644 --- a/openedx/features/survey_report/models.py +++ b/openedx/features/survey_report/models.py @@ -22,9 +22,7 @@ class SurveyReport(models.Model): default=dict, help_text="Extra information for instance data", ) - request_details = models.CharField(max_length=255, null=True, blank=True) created_at = models.DateTimeField(auto_now=True) - sent_at = models.DateTimeField(null=True, blank=True) class Meta: ordering = ["-created_at"] From ececb9574f66c3416318b8cd27596d8963f4befb Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 3 Nov 2022 15:17:09 -0500 Subject: [PATCH 09/22] test: rename mocks in command tests --- .../commands/tests/test_generate_report.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openedx/features/survey_report/management/commands/tests/test_generate_report.py b/openedx/features/survey_report/management/commands/tests/test_generate_report.py index 45f0d553c86d..84e7a38325e1 100644 --- a/openedx/features/survey_report/management/commands/tests/test_generate_report.py +++ b/openedx/features/survey_report/management/commands/tests/test_generate_report.py @@ -18,18 +18,18 @@ class GenerateReportTest(TestCase): @override_settings(SURVEY_REPORT_EXTRA_DATA={'extra_data': 'extra_data'}) @mock.patch('openedx.features.survey_report.queries.get_course_enrollments') @mock.patch('openedx.features.survey_report.queries.get_generated_certificates') - @mock.patch('openedx.features.survey_report.queries.get_learners_registered') - @mock.patch('openedx.features.survey_report.queries.get_currently_learners') + @mock.patch('openedx.features.survey_report.queries.get_registered_learners') + @mock.patch('openedx.features.survey_report.queries.get_recently_active_users') @mock.patch('openedx.features.survey_report.queries.get_unique_courses_offered') - def test_generate_report(self, mock_get_unique_courses_offered, mock_get_currently_learners, - mock_get_learners_registered, mock_get_generated_certificates, + def test_generate_report(self, mock_get_unique_courses_offered, mock_get_recently_active_users, + mock_get_registered_learners, mock_get_generated_certificates, mock_get_course_enrollments): """ Test that generate_report command creates a survey report. """ mock_get_unique_courses_offered.return_value = 1 - mock_get_currently_learners.return_value = 2 - mock_get_learners_registered.return_value = 3 + mock_get_recently_active_users.return_value = 2 + mock_get_registered_learners.return_value = 3 mock_get_generated_certificates.return_value = 4 mock_get_course_enrollments.return_value = 5 out = StringIO() From e57b10d8338915ba341d3db78e207d50614ba77c Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Fri, 4 Nov 2022 07:23:26 -0500 Subject: [PATCH 10/22] test: update test name --- openedx/features/survey_report/tests/test_query_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/features/survey_report/tests/test_query_methods.py b/openedx/features/survey_report/tests/test_query_methods.py index e7dcadc0626d..24749869ee4a 100644 --- a/openedx/features/survey_report/tests/test_query_methods.py +++ b/openedx/features/survey_report/tests/test_query_methods.py @@ -57,7 +57,7 @@ def test_get_unique_courses_offered(self): mock_datetime.now.return_value = datetime.now() assert get_unique_courses_offered() == 1 - def test_get_currently_learners(self): + def test_get_recently_active_users(self): """ Test that get_currently_learners returns the correct number of learners. """ From 614bf1c3b875b156c761537b1a384cf24a9019b4 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Tue, 8 Nov 2022 07:39:01 -0500 Subject: [PATCH 11/22] docs: add README file --- openedx/features/survey_report/README.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 openedx/features/survey_report/README.rst diff --git a/openedx/features/survey_report/README.rst b/openedx/features/survey_report/README.rst new file mode 100644 index 000000000000..5007244eff41 --- /dev/null +++ b/openedx/features/survey_report/README.rst @@ -0,0 +1,11 @@ +Survey Report +-------------------- +This django app was created for the purpose of aggregated, anonymized data +about Open edX courses at scale, so that we can begin to track the growth +and trends in Open edX usage over time, namely in the annual Open edX +Impact Report. + +You could find in this directory some methods to manage survey +reports, one command to generate the report, some queries to get the +information from database and one method to send the report to openedx +api. From c633c43078b83c92eda0ddd83b2d77bf05c83c3b Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Tue, 8 Nov 2022 07:48:24 -0500 Subject: [PATCH 12/22] docs: add selection criteria to get unique courses --- openedx/features/survey_report/queries.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index e1dffd83be02..90de274bbc8d 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -15,7 +15,8 @@ def get_unique_courses_offered() -> int: """ - Get total number of unique courses offered. + Get total number of unique course that started before today and have an open date, + or have not finished yet, whose number of enrollments is greater than 5. """ return CourseOverview.objects.annotate( count=Subquery( From d7d69341de7f7158e13a4e722a1968f794755148 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Wed, 9 Nov 2022 08:37:52 -0500 Subject: [PATCH 13/22] docs: update README Co-authored-by: David Ormsbee --- openedx/features/survey_report/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/features/survey_report/README.rst b/openedx/features/survey_report/README.rst index 5007244eff41..06c120ba9cb2 100644 --- a/openedx/features/survey_report/README.rst +++ b/openedx/features/survey_report/README.rst @@ -1,6 +1,6 @@ Survey Report -------------------- -This django app was created for the purpose of aggregated, anonymized data +This Django app was created for the purpose of gathering aggregated, anonymized data about Open edX courses at scale, so that we can begin to track the growth and trends in Open edX usage over time, namely in the annual Open edX Impact Report. From 5db5ef9407c40aa3774bc635eb0d09809d276acd Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Wed, 9 Nov 2022 09:03:21 -0500 Subject: [PATCH 14/22] test: remove useless mocks and use default modulestore --- .../survey_report/tests/test_query_methods.py | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/openedx/features/survey_report/tests/test_query_methods.py b/openedx/features/survey_report/tests/test_query_methods.py index 24749869ee4a..fd1a41f504f5 100644 --- a/openedx/features/survey_report/tests/test_query_methods.py +++ b/openedx/features/survey_report/tests/test_query_methods.py @@ -3,7 +3,6 @@ """ from datetime import datetime, timedelta -from unittest.mock import patch from common.djangoapps.student.tests.factories import CourseEnrollmentFactory, UserFactory from lms.djangoapps.grades.models import PersistentCourseGrade @@ -15,7 +14,7 @@ get_registered_learners, get_unique_courses_offered ) -from xmodule.modulestore import ModuleStoreEnum # lint-amnesty, pylint: disable=wrong-import-order + from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory # lint-amnesty, pylint: disable=wrong-import-order @@ -31,9 +30,9 @@ def setUp(self): Setup for users and courses. """ super().setUp() - self.store = modulestore()._get_modulestore_by_type(ModuleStoreEnum.Type.mongo) # lint-amnesty, pylint: disable=protected-access + self.store = modulestore() # lint-amnesty, pylint: disable=protected-access self.first_course = CourseFactory.create( - org="test", course="course1", display_name="run1", default_store=ModuleStoreEnum.Type.mongo + org="test", course="course1", display_name="run1" ) self.user = UserFactory.create(username='test_user', email='test@example.com', password='password') self.user1 = UserFactory.create(username='test_user1', email='test1@example.com', password='password') @@ -53,9 +52,7 @@ def test_get_unique_courses_offered(self): CourseEnrollmentFactory.create(user=self.user3, course_id=course_overview.id) CourseEnrollmentFactory.create(user=self.user4, course_id=course_overview.id) CourseEnrollmentFactory.create(user=self.user5, course_id=course_overview.id) - with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: - mock_datetime.now.return_value = datetime.now() - assert get_unique_courses_offered() == 1 + assert get_unique_courses_offered() == 1 def test_get_recently_active_users(self): """ @@ -67,9 +64,7 @@ def test_get_recently_active_users(self): self.user1.save() self.user2.last_login = datetime.now() - timedelta(weeks=4) self.user2.save() - with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: - mock_datetime.now.return_value = datetime.now() - assert get_recently_active_users(weeks=3) == 2 + assert get_recently_active_users(weeks=3) == 2 def test_get_learners_registered(self): """ @@ -90,10 +85,7 @@ def test_get_generated_certificates(self): "passed_timestamp": datetime.now(), } PersistentCourseGrade.update_or_create(**course_grade_params) - - with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: - mock_datetime.now.return_value = datetime.now() - assert get_generated_certificates() == 1 + assert get_generated_certificates() == 1 def test_get_course_enrollments(self): """ @@ -109,6 +101,4 @@ def test_get_course_enrollments(self): CourseEnrollmentFactory.create(user=self.user2, course_id=course_overview.id) CourseEnrollmentFactory.create(user=self.user3, course_id=course_overview.id) CourseEnrollmentFactory.create(user=self.user4, course_id=course_overview.id) - with patch('openedx.features.survey_report.queries.datetime') as mock_datetime: - mock_datetime.now.return_value = datetime.now() - assert get_course_enrollments() == 3 + assert get_course_enrollments() == 3 From 4a2117d4fe7e762272800b31ebdd93d390d8ba8c Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Wed, 9 Nov 2022 13:15:04 -0500 Subject: [PATCH 15/22] docs: change command error message --- .../survey_report/management/commands/generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/features/survey_report/management/commands/generate_report.py b/openedx/features/survey_report/management/commands/generate_report.py index e08d89ae4924..8965d6e5c224 100644 --- a/openedx/features/survey_report/management/commands/generate_report.py +++ b/openedx/features/survey_report/management/commands/generate_report.py @@ -23,6 +23,6 @@ def handle(self, *_args, **_options): try: generate_report() except Exception as error: - raise CommandError(f'An error has occurred while report was generating. {error}') from error + raise CommandError(f'An error has occurred while survey report was generating. {error}') from error self.stdout.write(self.style.SUCCESS('Survey report has been generated successfully.')) From f1a611ae76c60aa8cec950cb18c95808989c76f7 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 10 Nov 2022 10:39:32 -0500 Subject: [PATCH 16/22] docs: add docs decisions --- .../decisions/0001-addition-to-core-repo.rst | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 openedx/features/survey_report/docs/decisions/0001-addition-to-core-repo.rst diff --git a/openedx/features/survey_report/docs/decisions/0001-addition-to-core-repo.rst b/openedx/features/survey_report/docs/decisions/0001-addition-to-core-repo.rst new file mode 100644 index 000000000000..ec768426866b --- /dev/null +++ b/openedx/features/survey_report/docs/decisions/0001-addition-to-core-repo.rst @@ -0,0 +1,29 @@ +Addition of the Survey Report App to edx-platform +================================================= + +Status +------ +Accepted + +Context +------- +The transition to a more modular architecture for edx-platorm has been +strengthened by the acceptance of the `No new Django apps ADR`_. + +.. _No new Django apps ADR: https://github.com/openedx/edx-platform/tree/master/docs/decisions/0014-no-new-apps.rst + + +Rationale +--------- + +This feature was considered for inclusion into the edx-platform code because it +imports several models from the inner workings of the core functionality in +order to query them. This goes in accordance with the section further guidance +of the ADR. + + +Decision +-------- + +Locate the Survey Report Application in the edx-platform repository under +`openedx/features`. From 2c7de0ce339b34bf99c49e858bedcebfb8f98706 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 10 Nov 2022 12:02:24 -0500 Subject: [PATCH 17/22] docs: Update openedx/features/survey_report/management/commands/generate_report.py Co-authored-by: Maria Grimaldi --- .../survey_report/management/commands/generate_report.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openedx/features/survey_report/management/commands/generate_report.py b/openedx/features/survey_report/management/commands/generate_report.py index 8965d6e5c224..cb94f9ad2e97 100644 --- a/openedx/features/survey_report/management/commands/generate_report.py +++ b/openedx/features/survey_report/management/commands/generate_report.py @@ -15,7 +15,10 @@ class Command(BaseCommand): help = """ This command will create a new survey report using some - models to get the total number oof courses offered, currently active learners, + models to get: + - Total number of courses offered + - Currently active learners + ... learners ever registered, and generated certificates. """ From de046f3bec4f235de2115f5923ca5f329777aed7 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Thu, 10 Nov 2022 15:39:57 -0500 Subject: [PATCH 18/22] docs: add fields descriptions --- openedx/features/survey_report/models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/openedx/features/survey_report/models.py b/openedx/features/survey_report/models.py index 08faf19902e6..cb3353ac2fd9 100644 --- a/openedx/features/survey_report/models.py +++ b/openedx/features/survey_report/models.py @@ -11,6 +11,14 @@ class SurveyReport(models.Model): This model stores information to automate the way of gathering impact data from the openedx project. .. no_pii: + + fields: + - courses_offered: Total number of active unique courses. + - learner: Recently active users with login in some weeks. + - registered_learners: Total number of users ever registered in the platform. + - enrollments: Total number of active enrollments in the platform. + - generated_certificates: Total number of generated certificates. + - extra_data: Extra information that will be saved in the report, E.g: site_name, openedx-release. """ courses_offered = models.BigIntegerField() learners = models.BigIntegerField() From eacaeba8d67aa3100d77be3478a542555756a227 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Fri, 11 Nov 2022 10:17:27 -0500 Subject: [PATCH 19/22] docs: add logs for each query --- openedx/features/survey_report/queries.py | 30 +++++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index 90de274bbc8d..5c7a62f769ff 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -2,6 +2,7 @@ Queries to get data from database. """ +import logging from datetime import datetime, timedelta from django.contrib.auth.models import User # lint-amnesty, pylint: disable=imported-auth-user @@ -12,13 +13,15 @@ from lms.djangoapps.grades.models import PersistentCourseGrade from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +log = logging.getLogger(__name__) def get_unique_courses_offered() -> int: """ Get total number of unique course that started before today and have an open date, or have not finished yet, whose number of enrollments is greater than 5. """ - return CourseOverview.objects.annotate( + log.info("Getting the total number of unique courses offered...") + total = CourseOverview.objects.annotate( count=Subquery( CourseEnrollment.objects .filter(course_id=OuterRef('id')) @@ -31,6 +34,8 @@ def get_unique_courses_offered() -> int: .filter(Q(end__isnull=True) | Q(end__gt=datetime.now()))\ .using(read_replica_or_default())\ .count() + log.info("Getting the total number of unique courses offered... DONE") + return total def get_recently_active_users(weeks: int) -> int: @@ -40,33 +45,42 @@ def get_recently_active_users(weeks: int) -> int: Args: weeks (int): number of weeks since the last login to considerate as an active learner. """ - return User.objects.filter(last_login__gte=datetime.now() - timedelta(weeks=weeks))\ + log.info("Getting the total number of recently active users...") + total = User.objects.filter(last_login__gte=datetime.now() - timedelta(weeks=weeks))\ .using(read_replica_or_default())\ .count() - + log.info("Getting total number of recently active users... DONE") + return total def get_registered_learners() -> int: """ Get total number of active learners registered. """ - return User.objects.filter(is_active=True)\ + log.info("Getting the total number of ever registered learners...") + total = User.objects.filter(is_active=True)\ .using(read_replica_or_default())\ .count() - + log.info("Getting the total number of ever registered learners... DONE") + return total def get_generated_certificates() -> int: """ Get total number of generated certificates. """ - return PersistentCourseGrade.objects.filter(passed_timestamp__isnull=False)\ + log.info("Getting the total number of generated certificates...") + total = PersistentCourseGrade.objects.filter(passed_timestamp__isnull=False)\ .using(read_replica_or_default())\ .count() - + log.info("Getting the total number of generated certificates... DONE") + return total def get_course_enrollments() -> int: """ Get total number of enrollments from users that aren't staff. Course staff members will be included. """ - return CourseEnrollment.objects.filter(is_active=True, user__is_superuser=False, user__is_staff=False)\ + log.info("Getting the total number of course enrollments...") + total = CourseEnrollment.objects.filter(is_active=True, user__is_superuser=False, user__is_staff=False)\ .using(read_replica_or_default())\ .count() + log.info("Getting the total number of course enrollments... DONE") + return total From a603a8361b2415607b6d3fa17589da15cb4be61d Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Fri, 11 Nov 2022 10:28:53 -0500 Subject: [PATCH 20/22] style: add blank lines --- openedx/features/survey_report/queries.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index 5c7a62f769ff..648ce5aa9cb1 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -15,6 +15,7 @@ log = logging.getLogger(__name__) + def get_unique_courses_offered() -> int: """ Get total number of unique course that started before today and have an open date, @@ -52,6 +53,7 @@ def get_recently_active_users(weeks: int) -> int: log.info("Getting total number of recently active users... DONE") return total + def get_registered_learners() -> int: """ Get total number of active learners registered. @@ -63,6 +65,7 @@ def get_registered_learners() -> int: log.info("Getting the total number of ever registered learners... DONE") return total + def get_generated_certificates() -> int: """ Get total number of generated certificates. @@ -74,6 +77,7 @@ def get_generated_certificates() -> int: log.info("Getting the total number of generated certificates... DONE") return total + def get_course_enrollments() -> int: """ Get total number of enrollments from users that aren't staff. Course staff members will be included. From 83835688d483c60a3b67755203f7f9c8d11a7de0 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Fri, 11 Nov 2022 11:17:25 -0500 Subject: [PATCH 21/22] refactor: rename variables and add a constant for weeks --- openedx/features/survey_report/api.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openedx/features/survey_report/api.py b/openedx/features/survey_report/api.py index 327a99febff0..dc1f621a6aab 100644 --- a/openedx/features/survey_report/api.py +++ b/openedx/features/survey_report/api.py @@ -13,12 +13,14 @@ get_unique_courses_offered ) +MAX_WEEKS_SINCE_LAST_LOGIN: int = 4 + def generate_report() -> None: """ Generate a report with relevant data.""" courses_offered = get_unique_courses_offered() - learners = get_recently_active_users(weeks=3) - registered = get_registered_learners() + learners = get_recently_active_users(weeks=MAX_WEEKS_SINCE_LAST_LOGIN) + registered_learners = get_registered_learners() certificates = get_generated_certificates() enrollments = get_course_enrollments() extra_data = settings.SURVEY_REPORT_EXTRA_DATA @@ -26,7 +28,7 @@ def generate_report() -> None: survey_report = SurveyReport( courses_offered=courses_offered, learners=learners, - registered_learners=registered, + registered_learners=registered_learners, generated_certificates=certificates, enrollments=enrollments, extra_data=extra_data, From a26f45301f17091d5911822d7f7c59767109f056 Mon Sep 17 00:00:00 2001 From: Alejandro Cardenas Date: Sat, 12 Nov 2022 13:43:14 -0500 Subject: [PATCH 22/22] refactor: add constant MIN_ENROLLS_ACTIVE_COURSE --- openedx/features/survey_report/queries.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/openedx/features/survey_report/queries.py b/openedx/features/survey_report/queries.py index 648ce5aa9cb1..011a0d5b4790 100644 --- a/openedx/features/survey_report/queries.py +++ b/openedx/features/survey_report/queries.py @@ -15,11 +15,13 @@ log = logging.getLogger(__name__) +MIN_ENROLLS_ACTIVE_COURSE: int = 5 + def get_unique_courses_offered() -> int: """ Get total number of unique course that started before today and have an open date, - or have not finished yet, whose number of enrollments is greater than 5. + or have not finished yet, whose number of enrollments is greater than MIN_ENROLLS_ACTIVE_COURSE. """ log.info("Getting the total number of unique courses offered...") total = CourseOverview.objects.annotate( @@ -30,7 +32,7 @@ def get_unique_courses_offered() -> int: .annotate(count=Count('course_id')) .values('count') ))\ - .filter(count__gt=5)\ + .filter(count__gt=MIN_ENROLLS_ACTIVE_COURSE)\ .filter(start__lt=datetime.now())\ .filter(Q(end__isnull=True) | Q(end__gt=datetime.now()))\ .using(read_replica_or_default())\