feat: enhance cookie monitoring and logging

cms/djangoapps/contentstore/tests/test_course_listing.py

@@ -287,10 +287,10 @@ def test_course_listing_performance(self, store, courses_list_from_group_calls,
         courses_list, __ = _accessible_courses_list_from_groups(self.request)
         self.assertEqual(len(courses_list), USER_COURSES_COUNT)
 
-        with self.assertNumQueries(courses_list_from_group_calls, table_blacklist=WAFFLE_TABLES):
+        with self.assertNumQueries(courses_list_from_group_calls, table_ignorelist=WAFFLE_TABLES):
             _accessible_courses_list_from_groups(self.request)
 
-        with self.assertNumQueries(courses_list_calls, table_blacklist=WAFFLE_TABLES):
+        with self.assertNumQueries(courses_list_calls, table_ignorelist=WAFFLE_TABLES):
             _accessible_courses_iter_for_tests(self.request)
 
     def test_course_listing_errored_deleted_courses(self):

cms/djangoapps/contentstore/views/tests/test_course_index.py

@@ -380,7 +380,7 @@ def check_index_page_with_query_count(self, separate_archived_courses, org, mong
         """
         Checks the index page, and ensures the number of database queries is as expected.
         """
-        with self.assertNumQueries(sql_queries, table_blacklist=WAFFLE_TABLES):
+        with self.assertNumQueries(sql_queries, table_ignorelist=WAFFLE_TABLES):
             with check_mongo_calls(mongo_queries):
                 self.check_index_page(separate_archived_courses=separate_archived_courses, org=org)
 

cms/envs/common.py

@@ -763,8 +763,7 @@
     # Cookie monitoring
     'openedx.core.lib.request_utils.CookieMonitoringMiddleware',
 
-    # After cookie monitoring, but before anything else that looks at
-    # cookies, especially the session middleware
+    # Before anything that looks at cookies, especially the session middleware
     'openedx.core.djangoapps.cookie_metadata.middleware.CookieNameChange',
 
     'openedx.core.djangoapps.header_control.middleware.HeaderControlMiddleware',

lms/djangoapps/ccx/tests/test_field_override_performance.py

@@ -34,7 +34,7 @@
 from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES
 from openedx.features.content_type_gating.models import ContentTypeGatingConfig
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 
 @mock.patch.dict(
@@ -184,7 +184,7 @@ def instrument_course_progress_render(
             # can actually take affect.
             OverrideFieldData.provider_classes = None
 
-            with self.assertNumQueries(sql_queries, using='default', table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+            with self.assertNumQueries(sql_queries, using='default', table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
                 with self.assertNumQueries(0, using='student_module_history'):
                     with self.assertMongoCallCount(mongo_reads):
                         with self.assertXBlockInstantiations(1):

lms/djangoapps/certificates/apis/v0/tests/test_views.py

@@ -21,6 +21,7 @@
 from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory
 from openedx.core.djangoapps.user_api.tests.factories import UserPreferenceFactory
 from openedx.core.djangoapps.user_authn.tests.utils import JWT_AUTH_TYPES, AuthAndScopesTestMixin, AuthType
+from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES
 from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.modulestore.tests.factories import CourseFactory  # lint-amnesty, pylint: disable=wrong-import-order
 
@@ -308,7 +309,7 @@ def test_no_certificate(self):
     def test_query_counts(self):
         # Test student with no certificates
         student_no_cert = UserFactory.create(password=self.user_password)
-        with self.assertNumQueries(18):
+        with self.assertNumQueries(17, table_ignorelist=WAFFLE_TABLES):
             resp = self.get_response(
                 AuthType.jwt,
                 requesting_user=self.global_staff,
@@ -318,7 +319,7 @@ def test_query_counts(self):
             assert len(resp.data) == 0
 
         # Test student with 1 certificate
-        with self.assertNumQueries(12):
+        with self.assertNumQueries(12, table_ignorelist=WAFFLE_TABLES):
             resp = self.get_response(
                 AuthType.jwt,
                 requesting_user=self.global_staff,
@@ -358,7 +359,7 @@ def test_query_counts(self):
             download_url='www.google.com',
             grade="0.88",
         )
-        with self.assertNumQueries(12):
+        with self.assertNumQueries(12, table_ignorelist=WAFFLE_TABLES):
             resp = self.get_response(
                 AuthType.jwt,
                 requesting_user=self.global_staff,

lms/djangoapps/course_api/tests/test_views.py

@@ -26,6 +26,7 @@
 from common.djangoapps.student.auth import add_users
 from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole
 from common.djangoapps.student.tests.factories import AdminFactory
+from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES
 from openedx.core.lib.api.view_utils import LazySequence
 from openedx.features.content_type_gating.models import ContentTypeGatingConfig
 from openedx.features.course_duration_limits.models import CourseDurationLimitConfig
@@ -416,14 +417,14 @@ def test_too_many_courses(self):
         self.setup_user(self.audit_user)
 
         # These query counts were found empirically
-        query_counts = [54, 46, 46, 46, 46, 46, 46, 46, 46, 46, 16]
+        query_counts = [50, 46, 46, 46, 46, 46, 46, 46, 46, 46, 16]
         ordered_course_ids = sorted([str(cid) for cid in (course_ids + [c.id for c in self.courses])])
 
         self.clear_caches()
 
         for page in range(1, 12):
             RequestCache.clear_all_namespaces()
-            with self.assertNumQueries(query_counts[page - 1]):
+            with self.assertNumQueries(query_counts[page - 1], table_ignorelist=WAFFLE_TABLES):
                 response = self.verify_response(params={'page': page, 'page_size': 30})
 
                 assert 'results' in response.data

lms/djangoapps/courseware/tests/test_access.py

@@ -56,7 +56,7 @@
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.partitions.partitions import MINIMUM_STATIC_PARTITION_ID, Group, UserPartition  # lint-amnesty, pylint: disable=wrong-import-order
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 # pylint: disable=protected-access
 
@@ -849,5 +849,5 @@ def test_course_catalog_access_num_queries(self, user_attr_name, action, course_
                 num_queries = 0
 
         course_overview = CourseOverview.get_from_id(course.id)
-        with self.assertNumQueries(num_queries, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+        with self.assertNumQueries(num_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
             bool(access.has_access(user, action, course_overview, course_key=course.id))

lms/djangoapps/courseware/tests/test_course_info.py

@@ -32,7 +32,7 @@
 
 from .helpers import LoginEnrollmentTestCase
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 
 @override_waffle_flag(DISABLE_UNIFIED_COURSE_TAB_FLAG, active=True)
@@ -402,7 +402,7 @@ def fetch_course_info_with_queries(self, course, sql_queries, mongo_queries):
         and Mongo queries.
         """
         url = reverse('info', args=[str(course.id)])
-        with self.assertNumQueries(sql_queries, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+        with self.assertNumQueries(sql_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
             with check_mongo_calls(mongo_queries):
                 with mock.patch("openedx.core.djangoapps.theming.helpers.get_current_site", return_value=None):
                     resp = self.client.get(url)

lms/djangoapps/courseware/tests/test_views.py

@@ -126,7 +126,7 @@
 )
 from openedx.features.enterprise_support.tests.mixins.enterprise import EnterpriseTestConsentRequired
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 FEATURES_WITH_DISABLE_HONOR_CERTIFICATE = settings.FEATURES.copy()
 FEATURES_WITH_DISABLE_HONOR_CERTIFICATE['DISABLE_HONOR_CERTIFICATES'] = True
@@ -412,7 +412,7 @@ def test_index_query_counts(self):
         self.client.login(username=self.user.username, password=TEST_PASSWORD)
         CourseEnrollment.enroll(self.user, course.id)
 
-        with self.assertNumQueries(206, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+        with self.assertNumQueries(206, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
             with check_mongo_calls(3):
                 url = reverse(
                     'courseware_section',
@@ -1593,7 +1593,7 @@ def test_progress_queries_paced_courses(self, self_paced, query_count):
         # TODO: decrease query count as part of REVO-28
         ContentTypeGatingConfig.objects.create(enabled=True, enabled_as_of=datetime(2018, 1, 1))
         self.setup_course(self_paced=self_paced)
-        with self.assertNumQueries(query_count, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST), check_mongo_calls(2):
+        with self.assertNumQueries(query_count, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST), check_mongo_calls(2):
             self._get_progress_page()
 
     @patch.dict(settings.FEATURES, {'ASSUME_ZERO_GRADE_IF_ABSENT_FOR_ALL_TESTS': False})
@@ -1607,14 +1607,14 @@ def test_progress_queries(self, enable_waffle, initial, subsequent):
         self.setup_course()
         with override_waffle_switch(grades_waffle_switch(ASSUME_ZERO_GRADE_IF_ABSENT), active=enable_waffle):
             with self.assertNumQueries(
-                initial, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST
+                initial, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST
             ), check_mongo_calls(2):
                 self._get_progress_page()
 
             # subsequent accesses to the progress page require fewer queries.
             for _ in range(2):
                 with self.assertNumQueries(
-                    subsequent, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST
+                    subsequent, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST
                 ), check_mongo_calls(2):
                     self._get_progress_page()
 

lms/djangoapps/discussion/django_comment_client/base/tests.py

@@ -63,7 +63,7 @@
 
 CS_PREFIX = "http://localhost:4567/api/v1"
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 # pylint: disable=missing-docstring
 
@@ -397,7 +397,7 @@ def inner(self, default_store, module_count, mongo_calls, sql_queries, *args, **
             with modulestore().default_store(default_store):
                 self.set_up_course(module_count=module_count)
                 self.clear_caches()
-                with self.assertNumQueries(sql_queries, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+                with self.assertNumQueries(sql_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
                     with check_mongo_calls(mongo_calls):
                         func(self, *args, **kwargs)
         return inner

lms/djangoapps/discussion/tests/test_views.py

@@ -72,7 +72,7 @@
 
 log = logging.getLogger(__name__)
 
-QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
+QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
 
 
 class ViewsExceptionTestCase(UrlResetMixin, ModuleStoreTestCase):  # lint-amnesty, pylint: disable=missing-class-docstring
@@ -574,7 +574,7 @@ def call_single_thread():
             [num_cached_mongo_calls, AllowPlusOrMinusOneInt(num_cached_sql_queries)],
         ]
         for expected_mongo_calls, expected_sql_queries in cached_calls:
-            with self.assertNumQueries(expected_sql_queries, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+            with self.assertNumQueries(expected_sql_queries, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
                 with check_mongo_calls(expected_mongo_calls):
                     call_single_thread()
 

lms/envs/common.py

@@ -2067,8 +2067,7 @@ def _make_locale_paths(settings):  # pylint: disable=missing-function-docstring
     # Generate code ownership attributes. Keep this immediately after RequestCacheMiddleware.
     'edx_django_utils.monitoring.CodeOwnerMonitoringMiddleware',
 
-    # After cookie monitoring, but before anything else that looks at
-    # cookies, especially the session middleware
+    # Before anything that looks at cookies, especially the session middleware
     'openedx.core.djangoapps.cookie_metadata.middleware.CookieNameChange',
 
     # Monitoring and logging middleware

openedx/core/djangoapps/schedules/management/commands/tests/send_email_base.py

@@ -220,7 +220,7 @@ def test_schedule_bin(self, schedule_count, mock_attribute, mock_ace):
                         )
                         is_first_match = False
 
-                with self.assertNumQueries(expected_queries, table_blacklist=WAFFLE_TABLES):
+                with self.assertNumQueries(expected_queries, table_ignorelist=WAFFLE_TABLES):
                     self.task().apply(kwargs=dict(
                         site_id=self.site_config.site.id, target_day_str=target_day_str, day_offset=offset, bin_num=b,
                     ))
@@ -377,7 +377,7 @@ def test_multiple_target_schedules(self, mock_ace):
         # one query for course modes for the first schedule if we aren't checking the deadline for each course
         additional_course_queries = (num_courses * 2) - 1 if self.queries_deadline_for_each_course else 1
         expected_query_count = NUM_QUERIES_FIRST_MATCH + additional_course_queries
-        with self.assertNumQueries(expected_query_count, table_blacklist=WAFFLE_TABLES):
+        with self.assertNumQueries(expected_query_count, table_ignorelist=WAFFLE_TABLES):
             with patch.object(self.task, 'async_send_task') as mock_schedule_send:
                 self.task().apply(kwargs=dict(
                     site_id=self.site_config.site.id, target_day_str=serialize(target_day), day_offset=offset,
@@ -431,7 +431,7 @@ def _assert_template_for_offset(self, offset, message_count):  # lint-amnesty, p
                 else:
                     num_expected_queries += 1
 
-                with self.assertNumQueries(num_expected_queries, table_blacklist=WAFFLE_TABLES):
+                with self.assertNumQueries(num_expected_queries, table_ignorelist=WAFFLE_TABLES):
                     self.task().apply(kwargs=dict(
                         site_id=self.site_config.site.id, target_day_str=serialize(target_day), day_offset=offset,
                         bin_num=self._calculate_bin_for_user(user),

openedx/core/djangoapps/user_api/accounts/tests/test_views.py

@@ -24,6 +24,7 @@
 from openedx.core.djangoapps.user_api.accounts import ACCOUNT_VISIBILITY_PREF_KEY
 from openedx.core.djangoapps.user_api.models import UserPreference
 from openedx.core.djangoapps.user_api.preferences.api import set_user_preference
+from openedx.core.djangoapps.waffle_utils.testutils import WAFFLE_TABLES
 from openedx.core.djangolib.testing.utils import CacheIsolationTestCase, skip_unless_lms
 from openedx.features.name_affirmation_api.utils import get_name_affirmation_service
 
@@ -208,7 +209,7 @@ def _verify_get_own_username(self, queries, expected_status=200):
         """
         Internal helper to perform the actual assertion
         """
-        with self.assertNumQueries(queries):
+        with self.assertNumQueries(queries, table_ignorelist=WAFFLE_TABLES):
             response = self.send_get(self.client, expected_status=expected_status)
         if expected_status == 200:
             data = response.data
@@ -220,7 +221,7 @@ def test_get_username(self):
         Test that a client (logged in) can get her own username.
         """
         self.client.login(username=self.user.username, password=TEST_PASSWORD)
-        self._verify_get_own_username(17)
+        self._verify_get_own_username(16)
 
     def test_get_username_inactive(self):
         """
@@ -230,7 +231,7 @@ def test_get_username_inactive(self):
         self.client.login(username=self.user.username, password=TEST_PASSWORD)
         self.user.is_active = False
         self.user.save()
-        self._verify_get_own_username(17)
+        self._verify_get_own_username(16)
 
     def test_get_username_not_logged_in(self):
         """
@@ -239,7 +240,7 @@ def test_get_username_not_logged_in(self):
         """
 
         # verify that the endpoint is inaccessible when not logged in
-        self._verify_get_own_username(13, expected_status=401)
+        self._verify_get_own_username(12, expected_status=401)
 
 
 @ddt.ddt
@@ -256,7 +257,7 @@ class TestAccountsAPI(CacheIsolationTestCase, UserAPITestCase):
     """
 
     ENABLED_CACHES = ['default']
-    TOTAL_QUERY_COUNT = 26
+    TOTAL_QUERY_COUNT = 25
     FULL_RESPONSE_FIELD_COUNT = 30
 
     def setUp(self):
@@ -520,7 +521,7 @@ def test_get_account_different_user_visible(self):
         """
         self.different_client.login(username=self.different_user.username, password=TEST_PASSWORD)
         self.create_mock_profile(self.user)
-        with self.assertNumQueries(self._get_num_queries(self.TOTAL_QUERY_COUNT)):
+        with self.assertNumQueries(self._get_num_queries(self.TOTAL_QUERY_COUNT), table_ignorelist=WAFFLE_TABLES):
             response = self.send_get(self.different_client)
         self._verify_full_shareable_account_response(response, account_privacy=ALL_USERS_VISIBILITY)
 
@@ -535,7 +536,7 @@ def test_get_account_different_user_private(self):
         """
         self.different_client.login(username=self.different_user.username, password=TEST_PASSWORD)
         self.create_mock_profile(self.user)
-        with self.assertNumQueries(self._get_num_queries(self.TOTAL_QUERY_COUNT)):
+        with self.assertNumQueries(self._get_num_queries(self.TOTAL_QUERY_COUNT), table_ignorelist=WAFFLE_TABLES):
             response = self.send_get(self.different_client)
         self._verify_private_account_response(response)
 
@@ -660,7 +661,7 @@ def verify_get_own_information(queries):
             """
             Internal helper to perform the actual assertions
             """
-            with self.assertNumQueries(queries):
+            with self.assertNumQueries(queries, table_ignorelist=WAFFLE_TABLES):
                 response = self.send_get(self.client)
             data = response.data
             assert self.FULL_RESPONSE_FIELD_COUNT == len(data)
@@ -686,7 +687,7 @@ def verify_get_own_information(queries):
             assert data['accomplishments_shared'] is False
 
         self.client.login(username=self.user.username, password=TEST_PASSWORD)
-        verify_get_own_information(self._get_num_queries(24))
+        verify_get_own_information(self._get_num_queries(23))
 
         # Now make sure that the user can get the same information, even if not active
         self.user.is_active = False
@@ -706,7 +707,7 @@ def test_get_account_empty_string(self):
         legacy_profile.save()
 
         self.client.login(username=self.user.username, password=TEST_PASSWORD)
-        with self.assertNumQueries(self._get_num_queries(24)):
+        with self.assertNumQueries(self._get_num_queries(23), table_ignorelist=WAFFLE_TABLES):
             response = self.send_get(self.client)
         for empty_field in ("level_of_education", "gender", "country", "state", "bio",):
             assert response.data[empty_field] is None

openedx/core/djangoapps/waffle_utils/testutils.py

@@ -4,8 +4,8 @@
 
 # Can be used with FilteredQueryCountMixin.assertNumQueries() to blacklist
 # waffle tables. For example:
-#   QUERY_COUNT_TABLE_BLACKLIST = WAFFLE_TABLES
-#   with self.assertNumQueries(6, table_blacklist=QUERY_COUNT_TABLE_BLACKLIST):
+#   QUERY_COUNT_TABLE_IGNORELIST = WAFFLE_TABLES
+#   with self.assertNumQueries(6, table_ignorelist=QUERY_COUNT_TABLE_IGNORELIST):
 WAFFLE_TABLES = [
     "waffle_utils_waffleflagcourseoverridemodel",
     "waffle_utils_waffleflagorgoverridemodel",