Capa problem type filtering via search index (SOL-286)

cms/djangoapps/contentstore/tests/test_libraries.py

@@ -385,6 +385,7 @@ def test_refreshes_children_if_libraries_change(self):
         html_block = modulestore().get_item(lc_block.children[0])
         self.assertEqual(html_block.data, data2)
 
+    @patch("xmodule.library_tools.SearchEngine.get_search_engine", Mock(return_value=None))
     def test_refreshes_children_if_capa_type_change(self):
         """ Tests that children are automatically refreshed if capa type field changes """
         name1, name2 = "Option Problem", "Multiple Choice Problem"

cms/djangoapps/contentstore/views/course.py

@@ -22,7 +22,7 @@
 from xmodule.course_module import DEFAULT_START_DATE
 from xmodule.error_module import ErrorDescriptor
 from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer, SearchIndexingError
+from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError
 from xmodule.contentstore.content import StaticContent
 from xmodule.tabs import PDFTextbookTabs
 from xmodule.partitions.partitions import UserPartition
@@ -137,7 +137,8 @@ def reindex_course_and_check_access(course_key, user):
     """
     if not has_course_author_access(user, course_key):
         raise PermissionDenied()
-    return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key)
+    indexer = get_indexer_for_location(course_key)
+    return indexer.do_course_reindex(modulestore(), course_key)
 
 
 @login_required

cms/djangoapps/contentstore/views/tests/test_course_index.py

@@ -15,11 +15,10 @@
 from course_action_state.models import CourseRerunState
 from util.date_utils import get_default_time_display
 from xmodule.modulestore import ModuleStoreEnum
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
+from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.modulestore.django import modulestore
 from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory
-from xmodule.modulestore.courseware_index import SearchIndexingError
 from opaque_keys.edx.locator import CourseLocator
 from student.tests.factories import UserFactory
 from course_action_state.managers import CourseRerunUIStateManager
@@ -385,6 +384,20 @@ def setUp(self):
 
         self.addCleanup(os.remove, self.TEST_INDEX_FILENAME)
 
+    def _perform_search(self, query="unique"):
+        """ Performs search """
+        return perform_search(
+            query,
+            user=self.user,  # pylint: disable=no-member
+            size=10,
+            from_=0,
+            course_id=unicode(self.course.id))
+
+    def _do_reindex(self):
+        """ Reindexes course """
+        indexer = get_indexer_for_location(self.course.id)
+        return indexer.do_course_reindex(modulestore(), self.course.id)
+
     def test_reindex_course(self):
         """
         Verify that course gets reindexed.
@@ -446,12 +459,7 @@ def test_reindex_json_responses(self):
         Test json response with real data
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # Start manual reindex
@@ -464,12 +472,7 @@ def test_reindex_json_responses(self):
         reindex_course_and_check_access(self.course.id, self.user)
 
         # Check results indexed now
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['total'], 1)
 
     @mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary')
@@ -478,12 +481,7 @@ def test_reindex_video_error_json_responses(self, mock_index_dictionary):
         Test json response with mocked error data for video
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
@@ -500,12 +498,7 @@ def test_reindex_html_error_json_responses(self, mock_index_dictionary):
         Test json response with mocked error data for html
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
@@ -522,12 +515,7 @@ def test_reindex_seq_error_json_responses(self, mock_index_dictionary):
         Test json response with mocked error data for sequence
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
@@ -562,30 +550,20 @@ def test_indexing_responses(self):
         Test add_to_search_index response with real data
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # Start manual reindex
-        CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+        self._do_reindex()
 
         self.html.display_name = "My expanded HTML"
         modulestore().update_item(self.html, ModuleStoreEnum.UserID.test)
 
         # Start manual reindex
-        CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+        self._do_reindex()
 
         # Check results indexed now
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['total'], 1)
 
     @mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary')
@@ -594,75 +572,56 @@ def test_indexing_video_error_responses(self, mock_index_dictionary):
         Test add_to_search_index response with mocked error data for video
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
-        err = Exception
-        mock_index_dictionary.return_value = err
+        mock_index_dictionary.return_value = Exception
 
         # Start manual reindex and check error in response
         with self.assertRaises(SearchIndexingError):
-            CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+            self._do_reindex()
 
     @mock.patch('xmodule.html_module.HtmlDescriptor.index_dictionary')
     def test_indexing_html_error_responses(self, mock_index_dictionary):
         """
         Test add_to_search_index response with mocked error data for html
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
-        err = Exception
-        mock_index_dictionary.return_value = err
+        mock_index_dictionary.return_value = Exception
 
         # Start manual reindex and check error in response
         with self.assertRaises(SearchIndexingError):
-            CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+            self._do_reindex()
 
     @mock.patch('xmodule.seq_module.SequenceDescriptor.index_dictionary')
     def test_indexing_seq_error_responses(self, mock_index_dictionary):
         """
         Test add_to_search_index response with mocked error data for sequence
         """
         # Check results not indexed
-        response = perform_search(
-            "unique",
-            user=self.user,
-            size=10,
-            from_=0,
-            course_id=unicode(self.course.id))
+        response = self._perform_search()
         self.assertEqual(response['results'], [])
 
         # set mocked exception response
-        err = Exception
-        mock_index_dictionary.return_value = err
+        mock_index_dictionary.return_value = Exception
 
         # Start manual reindex and check error in response
         with self.assertRaises(SearchIndexingError):
-            CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+            self._do_reindex()
 
     @mock.patch('xmodule.modulestore.mongo.base.MongoModuleStore.get_course')
     def test_indexing_no_item(self, mock_get_course):
         """
         Test system logs an error if no item found.
         """
         # set mocked exception response
-        err = ItemNotFoundError
-        mock_get_course.return_value = err
+        mock_get_course.return_value = ItemNotFoundError
 
         # Start manual reindex and check error in response
         with self.assertRaises(SearchIndexingError):
-            CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+            self._do_reindex()

common/lib/xmodule/xmodule/capa_module.py

@@ -113,6 +113,7 @@ class CapaDescriptor(CapaFields, RawDescriptor):
     Module implementing problems in the LON-CAPA format,
     as implemented by capa.capa_problem
     """
+    INDEX_CONTENT_TYPE = 'CAPA'
 
     module_class = CapaModule
 
@@ -186,6 +187,21 @@ def problem_types(self):
         registered_tags = responsetypes.registry.registered_tags()
         return set([node.tag for node in tree.iter() if node.tag in registered_tags])
 
+    def index_dictionary(self):
+        """
+        Return dictionary prepared with module content and type for indexing.
+        """
+        result = super(CapaDescriptor, self).index_dictionary()
+        if not result:
+            result = {}
+        index = {
+            'content_type': self.INDEX_CONTENT_TYPE,
+            'problem_types': list(self.problem_types),
+            "display_name": self.display_name
+        }
+        result.update(index)
+        return result
+
     # Proxy to CapaModule for access to any of its attributes
     answer_available = module_attr('answer_available')
     check_button_name = module_attr('check_button_name')

common/lib/xmodule/xmodule/library_tools.py

@@ -2,7 +2,8 @@
 XBlock runtime services for LibraryContentModule
 """
 from django.core.exceptions import PermissionDenied
-from opaque_keys.edx.locator import LibraryLocator
+from opaque_keys.edx.locator import LibraryLocator, LibraryUsageLocator
+from search.search_engine_base import SearchEngine
 from xmodule.library_content_module import ANY_CAPA_TYPE_VALUE
 from xmodule.modulestore.exceptions import ItemNotFoundError
 from xmodule.capa_module import CapaDescriptor
@@ -82,13 +83,24 @@ def summarize_block(usage_key):
             result_json.append(info)
         return result_json
 
+    def _problem_type_filter(self, library, capa_type):
+        """ Filters library children by capa type"""
+        search_engine = SearchEngine.get_search_engine(index="library_index")
+        if search_engine:
+            filter_clause = {
+                "content_type": CapaDescriptor.INDEX_CONTENT_TYPE,
+                "problem_types": capa_type
+            }
+            search_result = search_engine.search(field_dictionary=filter_clause)
+            results = search_result.get('results', [])
+            return [LibraryUsageLocator.from_string(item['data']['id']) for item in results]
+        else:
+            return [key for key in library.children if self._filter_child(key, capa_type)]
+
     def _filter_child(self, usage_key, capa_type):
         """
         Filters children by CAPA problem type, if configured
         """
-        if capa_type == ANY_CAPA_TYPE_VALUE:
-            return True
-
         if usage_key.block_type != "problem":
             return False
 
@@ -131,7 +143,7 @@ def update_children(self, dest_block, user_id, user_perms=None):
         filter_children = (dest_block.capa_type != ANY_CAPA_TYPE_VALUE)
         if filter_children:
             # Apply simple filtering based on CAPA problem types:
-            source_blocks.extend([key for key in library.children if self._filter_child(key, dest_block.capa_type)])
+            source_blocks.extend(self._problem_type_filter(library, dest_block.capa_type))
         else:
             source_blocks.extend(library.children)