diff --git a/cms/djangoapps/contentstore/tests/test_libraries.py b/cms/djangoapps/contentstore/tests/test_libraries.py
index 0ff0bfabce16..cdc8d051dca0 100644
--- a/cms/djangoapps/contentstore/tests/test_libraries.py
+++ b/cms/djangoapps/contentstore/tests/test_libraries.py
@@ -385,6 +385,7 @@ def test_refreshes_children_if_libraries_change(self):
html_block = modulestore().get_item(lc_block.children[0])
self.assertEqual(html_block.data, data2)
+ @patch("xmodule.library_tools.SearchEngine.get_search_engine", Mock(return_value=None))
def test_refreshes_children_if_capa_type_change(self):
""" Tests that children are automatically refreshed if capa type field changes """
name1, name2 = "Option Problem", "Multiple Choice Problem"
diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py
index 7db80dc44f6d..cafd74d09006 100644
--- a/cms/djangoapps/contentstore/views/course.py
+++ b/cms/djangoapps/contentstore/views/course.py
@@ -22,7 +22,7 @@
from xmodule.course_module import DEFAULT_START_DATE
from xmodule.error_module import ErrorDescriptor
from xmodule.modulestore.django import modulestore
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer, SearchIndexingError
+from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError
from xmodule.contentstore.content import StaticContent
from xmodule.tabs import PDFTextbookTabs
from xmodule.partitions.partitions import UserPartition
@@ -137,7 +137,8 @@ def reindex_course_and_check_access(course_key, user):
"""
if not has_course_author_access(user, course_key):
raise PermissionDenied()
- return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key)
+ indexer = get_indexer_for_location(course_key)
+ return indexer.do_course_reindex(modulestore(), course_key)
@login_required
diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py
index 6c110839663a..9d154f41cef9 100644
--- a/cms/djangoapps/contentstore/views/tests/test_course_index.py
+++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py
@@ -15,11 +15,10 @@
from course_action_state.models import CourseRerunState
from util.date_utils import get_default_time_display
from xmodule.modulestore import ModuleStoreEnum
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
+from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory
-from xmodule.modulestore.courseware_index import SearchIndexingError
from opaque_keys.edx.locator import CourseLocator
from student.tests.factories import UserFactory
from course_action_state.managers import CourseRerunUIStateManager
@@ -385,6 +384,20 @@ def setUp(self):
self.addCleanup(os.remove, self.TEST_INDEX_FILENAME)
+ def _perform_search(self, query="unique"):
+ """ Performs search """
+ return perform_search(
+ query,
+ user=self.user, # pylint: disable=no-member
+ size=10,
+ from_=0,
+ course_id=unicode(self.course.id))
+
+ def _do_reindex(self):
+ """ Reindexes course """
+ indexer = get_indexer_for_location(self.course.id)
+ return indexer.do_course_reindex(modulestore(), self.course.id)
+
def test_reindex_course(self):
"""
Verify that course gets reindexed.
@@ -446,12 +459,7 @@ def test_reindex_json_responses(self):
Test json response with real data
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# Start manual reindex
@@ -464,12 +472,7 @@ def test_reindex_json_responses(self):
reindex_course_and_check_access(self.course.id, self.user)
# Check results indexed now
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['total'], 1)
@mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary')
@@ -478,12 +481,7 @@ def test_reindex_video_error_json_responses(self, mock_index_dictionary):
Test json response with mocked error data for video
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
@@ -500,12 +498,7 @@ def test_reindex_html_error_json_responses(self, mock_index_dictionary):
Test json response with mocked error data for html
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
@@ -522,12 +515,7 @@ def test_reindex_seq_error_json_responses(self, mock_index_dictionary):
Test json response with mocked error data for sequence
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
@@ -562,30 +550,20 @@ def test_indexing_responses(self):
Test add_to_search_index response with real data
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# Start manual reindex
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
self.html.display_name = "My expanded HTML"
modulestore().update_item(self.html, ModuleStoreEnum.UserID.test)
# Start manual reindex
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
# Check results indexed now
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['total'], 1)
@mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary')
@@ -594,21 +572,15 @@ def test_indexing_video_error_responses(self, mock_index_dictionary):
Test add_to_search_index response with mocked error data for video
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
- err = Exception
- mock_index_dictionary.return_value = err
+ mock_index_dictionary.return_value = Exception
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
@mock.patch('xmodule.html_module.HtmlDescriptor.index_dictionary')
def test_indexing_html_error_responses(self, mock_index_dictionary):
@@ -616,21 +588,15 @@ def test_indexing_html_error_responses(self, mock_index_dictionary):
Test add_to_search_index response with mocked error data for html
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
- err = Exception
- mock_index_dictionary.return_value = err
+ mock_index_dictionary.return_value = Exception
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
@mock.patch('xmodule.seq_module.SequenceDescriptor.index_dictionary')
def test_indexing_seq_error_responses(self, mock_index_dictionary):
@@ -638,21 +604,15 @@ def test_indexing_seq_error_responses(self, mock_index_dictionary):
Test add_to_search_index response with mocked error data for sequence
"""
# Check results not indexed
- response = perform_search(
- "unique",
- user=self.user,
- size=10,
- from_=0,
- course_id=unicode(self.course.id))
+ response = self._perform_search()
self.assertEqual(response['results'], [])
# set mocked exception response
- err = Exception
- mock_index_dictionary.return_value = err
+ mock_index_dictionary.return_value = Exception
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
@mock.patch('xmodule.modulestore.mongo.base.MongoModuleStore.get_course')
def test_indexing_no_item(self, mock_get_course):
@@ -660,9 +620,8 @@ def test_indexing_no_item(self, mock_get_course):
Test system logs an error if no item found.
"""
# set mocked exception response
- err = ItemNotFoundError
- mock_get_course.return_value = err
+ mock_get_course.return_value = ItemNotFoundError
# Start manual reindex and check error in response
with self.assertRaises(SearchIndexingError):
- CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id)
+ self._do_reindex()
diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py
index 7364730815ec..d647aa8c0ddf 100644
--- a/common/lib/xmodule/xmodule/capa_module.py
+++ b/common/lib/xmodule/xmodule/capa_module.py
@@ -113,6 +113,7 @@ class CapaDescriptor(CapaFields, RawDescriptor):
Module implementing problems in the LON-CAPA format,
as implemented by capa.capa_problem
"""
+ INDEX_CONTENT_TYPE = 'CAPA'
module_class = CapaModule
@@ -186,6 +187,21 @@ def problem_types(self):
registered_tags = responsetypes.registry.registered_tags()
return set([node.tag for node in tree.iter() if node.tag in registered_tags])
+ def index_dictionary(self):
+ """
+ Return dictionary prepared with module content and type for indexing.
+ """
+ result = super(CapaDescriptor, self).index_dictionary()
+ if not result:
+ result = {}
+ index = {
+ 'content_type': self.INDEX_CONTENT_TYPE,
+ 'problem_types': list(self.problem_types),
+ "display_name": self.display_name
+ }
+ result.update(index)
+ return result
+
# Proxy to CapaModule for access to any of its attributes
answer_available = module_attr('answer_available')
check_button_name = module_attr('check_button_name')
diff --git a/common/lib/xmodule/xmodule/library_tools.py b/common/lib/xmodule/xmodule/library_tools.py
index 40afbbb549e1..ee657f619f02 100644
--- a/common/lib/xmodule/xmodule/library_tools.py
+++ b/common/lib/xmodule/xmodule/library_tools.py
@@ -2,7 +2,8 @@
XBlock runtime services for LibraryContentModule
"""
from django.core.exceptions import PermissionDenied
-from opaque_keys.edx.locator import LibraryLocator
+from opaque_keys.edx.locator import LibraryLocator, LibraryUsageLocator
+from search.search_engine_base import SearchEngine
from xmodule.library_content_module import ANY_CAPA_TYPE_VALUE
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.capa_module import CapaDescriptor
@@ -82,13 +83,24 @@ def summarize_block(usage_key):
result_json.append(info)
return result_json
+ def _problem_type_filter(self, library, capa_type):
+ """ Filters library children by capa type"""
+ search_engine = SearchEngine.get_search_engine(index="library_index")
+ if search_engine:
+ filter_clause = {
+ "content_type": CapaDescriptor.INDEX_CONTENT_TYPE,
+ "problem_types": capa_type
+ }
+ search_result = search_engine.search(field_dictionary=filter_clause)
+ results = search_result.get('results', [])
+ return [LibraryUsageLocator.from_string(item['data']['id']) for item in results]
+ else:
+ return [key for key in library.children if self._filter_child(key, capa_type)]
+
def _filter_child(self, usage_key, capa_type):
"""
Filters children by CAPA problem type, if configured
"""
- if capa_type == ANY_CAPA_TYPE_VALUE:
- return True
-
if usage_key.block_type != "problem":
return False
@@ -131,7 +143,7 @@ def update_children(self, dest_block, user_id, user_perms=None):
filter_children = (dest_block.capa_type != ANY_CAPA_TYPE_VALUE)
if filter_children:
# Apply simple filtering based on CAPA problem types:
- source_blocks.extend([key for key in library.children if self._filter_child(key, dest_block.capa_type)])
+ source_blocks.extend(self._problem_type_filter(library, dest_block.capa_type))
else:
source_blocks.extend(library.children)
diff --git a/common/lib/xmodule/xmodule/modulestore/courseware_index.py b/common/lib/xmodule/xmodule/modulestore/courseware_index.py
deleted file mode 100644
index ff55b1a721e2..000000000000
--- a/common/lib/xmodule/xmodule/modulestore/courseware_index.py
+++ /dev/null
@@ -1,176 +0,0 @@
-""" Code to allow module store to interface with courseware index """
-from __future__ import absolute_import
-
-import logging
-
-from django.utils.translation import ugettext as _
-from opaque_keys.edx.locator import CourseLocator
-from search.search_engine_base import SearchEngine
-from eventtracking import tracker
-
-from . import ModuleStoreEnum
-from .exceptions import ItemNotFoundError
-
-
-# Use default index and document names for now
-INDEX_NAME = "courseware_index"
-DOCUMENT_TYPE = "courseware_content"
-
-log = logging.getLogger('edx.modulestore')
-
-
-class SearchIndexingError(Exception):
- """ Indicates some error(s) occured during indexing """
-
- def __init__(self, message, error_list):
- super(SearchIndexingError, self).__init__(message)
- self.error_list = error_list
-
-
-class CoursewareSearchIndexer(object):
- """
- Class to perform indexing for courseware search from different modulestores
- """
-
- @staticmethod
- def add_to_search_index(modulestore, location, delete=False, raise_on_error=False):
- """
- Add to courseware search index from given location and its children
- """
- error_list = []
- indexed_count = 0
- # TODO - inline for now, need to move this out to a celery task
- searcher = SearchEngine.get_search_engine(INDEX_NAME)
- if not searcher:
- return
-
- if isinstance(location, CourseLocator):
- course_key = location
- else:
- course_key = location.course_key
-
- location_info = {
- "course": unicode(course_key),
- }
-
- def _fetch_item(item_location):
- """ Fetch the item from the modulestore location, log if not found, but continue """
- try:
- if isinstance(item_location, CourseLocator):
- item = modulestore.get_course(item_location)
- else:
- item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only)
- except ItemNotFoundError:
- log.warning('Cannot find: %s', item_location)
- return None
-
- return item
-
- def index_item_location(item_location, current_start_date):
- """ add this item to the search index """
- item = _fetch_item(item_location)
- if not item:
- return
-
- is_indexable = hasattr(item, "index_dictionary")
- # if it's not indexable and it does not have children, then ignore
- if not is_indexable and not item.has_children:
- return
-
- # if it has a defined start, then apply it and to it's children
- if item.start and (not current_start_date or item.start > current_start_date):
- current_start_date = item.start
-
- if item.has_children:
- for child_loc in item.children:
- index_item_location(child_loc, current_start_date)
-
- item_index = {}
- item_index_dictionary = item.index_dictionary() if is_indexable else None
-
- # if it has something to add to the index, then add it
- if item_index_dictionary:
- try:
- item_index.update(location_info)
- item_index.update(item_index_dictionary)
- item_index['id'] = unicode(item.scope_ids.usage_id)
- if current_start_date:
- item_index['start_date'] = current_start_date
-
- searcher.index(DOCUMENT_TYPE, item_index)
- except Exception as err: # pylint: disable=broad-except
- # broad exception so that index operation does not fail on one item of many
- log.warning('Could not index item: %s - %s', item_location, unicode(err))
- error_list.append(_('Could not index item: {}').format(item_location))
-
- def remove_index_item_location(item_location):
- """ remove this item from the search index """
- item = _fetch_item(item_location)
- if item:
- if item.has_children:
- for child_loc in item.children:
- remove_index_item_location(child_loc)
-
- searcher.remove(DOCUMENT_TYPE, unicode(item.scope_ids.usage_id))
-
- try:
- if delete:
- remove_index_item_location(location)
- else:
- index_item_location(location, None)
- indexed_count += 1
- except Exception as err: # pylint: disable=broad-except
- # broad exception so that index operation does not prevent the rest of the application from working
- log.exception(
- "Indexing error encountered, courseware index may be out of date %s - %s",
- course_key,
- unicode(err)
- )
- error_list.append(_('General indexing error occurred'))
-
- if raise_on_error and error_list:
- raise SearchIndexingError(_('Error(s) present during indexing'), error_list)
-
- return indexed_count
-
- @classmethod
- def do_publish_index(cls, modulestore, location, delete=False, raise_on_error=False):
- """
- Add to courseware search index published section and children
- """
- indexed_count = cls.add_to_search_index(modulestore, location, delete, raise_on_error)
- cls._track_index_request('edx.course.index.published', indexed_count, str(location))
- return indexed_count
-
- @classmethod
- def do_course_reindex(cls, modulestore, course_key):
- """
- (Re)index all content within the given course
- """
- indexed_count = cls.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True)
- cls._track_index_request('edx.course.index.reindexed', indexed_count)
- return indexed_count
-
- @staticmethod
- def _track_index_request(event_name, indexed_count, location=None):
- """Track content index requests.
-
- Arguments:
- location (str): The ID of content to be indexed.
- event_name (str): Name of the event to be logged.
- Returns:
- None
-
- """
- data = {
- "indexed_count": indexed_count,
- 'category': 'courseware_index',
- }
-
- if location:
- data['location_id'] = location
-
- tracker.emit(
- event_name,
- data
- )
diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
index 5c30f5e379e3..731b3d235386 100644
--- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
+++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py
@@ -12,7 +12,7 @@
from opaque_keys.edx.locations import Location
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import ModuleStoreEnum
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
+from xmodule.modulestore.search_index import get_indexer_for_location
from xmodule.modulestore.exceptions import (
ItemNotFoundError, DuplicateItemError, DuplicateCourseError, InvalidBranchSetting
)
@@ -469,6 +469,11 @@ def update_item(self, xblock, user_id, allow_not_found=False, force=False, isPub
xblock.location = draft_loc
super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found, isPublish=isPublish)
+
+ indexer = get_indexer_for_location(xblock.location)
+ if indexer.index_on_update:
+ indexer.add_to_search_index(self, xblock.location)
+
return wrap_draft(xblock)
def delete_item(self, location, user_id, revision=None, **kwargs):
@@ -552,7 +557,9 @@ def delete_item(self, location, user_id, revision=None, **kwargs):
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
- CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
+ indexer = get_indexer_for_location(location)
+ if indexer.index_on_delete:
+ indexer.add_to_search_index(self, location, delete=True)
def _delete_subtree(self, location, as_functions, draft_only=False):
"""
@@ -732,7 +739,9 @@ def _internal_depth_first(item_location, is_root):
self.signal_handler.send("course_published", course_key=course_key)
# Now it's been published, add the object to the courseware search index so that it appears in search results
- CoursewareSearchIndexer.do_publish_index(self, location)
+ indexer = get_indexer_for_location(location)
+ if indexer.index_on_publish:
+ indexer.do_publish_index(self, location)
return self.get_item(as_published(location))
diff --git a/common/lib/xmodule/xmodule/modulestore/search_index.py b/common/lib/xmodule/xmodule/modulestore/search_index.py
new file mode 100644
index 000000000000..5852a521093b
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/search_index.py
@@ -0,0 +1,264 @@
+""" Code to allow module store to interface with courseware index """
+from __future__ import absolute_import
+
+import logging
+
+from django.utils.translation import ugettext as _
+from opaque_keys.edx.locator import CourseLocator, LibraryLocator
+from search.search_engine_base import SearchEngine
+from eventtracking import tracker
+
+from . import ModuleStoreEnum
+from .exceptions import ItemNotFoundError
+
+
+# Use default index and document names for now
+
+
+log = logging.getLogger('edx.modulestore')
+
+
+def get_indexer_for_location(locator):
+ """
+ Initializes correct indexer for given location.
+
+ Arguments:
+ locator BlockLocatorBase: XBlock locator
+ """
+ if isinstance(locator, CourseLocator):
+ return CoursewareSearchIndexer()
+ elif isinstance(locator, LibraryLocator):
+ return LibrarySearchIndexer()
+ return get_indexer_for_location(locator.course_key)
+
+
+class SearchIndexingError(Exception):
+ """ Indicates some error(s) occured during indexing """
+
+ def __init__(self, message, error_list):
+ super(SearchIndexingError, self).__init__(message)
+ self.error_list = error_list
+
+
+class SearchIndexerBase(object):
+ """
+ Base class to perform XBlock indexing from different modulestores
+ """
+ INDEX_NAME = None
+ DOCUMENT_TYPE = None
+
+ index_on_create = True
+ index_on_update = True
+ index_on_delete = True
+ index_on_publish = True
+
+ def _get_structure_key(self, location):
+ """ Gets structure key from location """
+ return location.course_key
+
+ def _get_location_info(self, structure_key):
+ """ Builds location info dictionary """
+ return {"course": unicode(structure_key)}
+
+ def _fetch_item(self, modulestore, item_location): # pylint: disable=unused-argument
+ """ Fetch the item from the modulestore location, log if not found, but continue """
+ raise NotImplementedError()
+
+ def _id_modifier(self, usage_id):
+ """ Modifies usage_id to submit to index """
+ return usage_id
+
+ @classmethod
+ def _track_index_request(cls, event_name, indexed_count, location=None):
+ """Track content index requests.
+
+ def add_to_search_index(self, modulestore, location, delete=False, raise_on_error=False):
+ Arguments:
+ location (str): The ID of content to be indexed.
+ event_name (str): Name of the event to be logged.
+ Returns:
+ None
+
+ """
+ data = {
+ "indexed_count": indexed_count,
+ 'category': cls.INDEX_NAME,
+ }
+
+ if location:
+ data['location_id'] = location
+
+ tracker.emit(
+ event_name,
+ data
+ )
+
+ def do_publish_index(self, modulestore, location, delete=False, raise_on_error=False):
+ """
+ Add to search index published section and children
+ """
+ indexed_count = self.add_to_search_index(modulestore, location, delete, raise_on_error)
+ self._track_index_request('edx.course.index.published', indexed_count, str(location))
+ return indexed_count
+
+ def add_to_search_index(self, modulestore, location, delete=False, raise_on_error=False):
+ """
+ Add to courseware search index from given location and its children
+ """
+ error_list = []
+ indexed_count = 0
+ # TODO - inline for now, need to move this out to a celery task
+ searcher = SearchEngine.get_search_engine(self.INDEX_NAME)
+ if not searcher:
+ return
+
+ structure_key = self._get_structure_key(location)
+ location_info = self._get_location_info(structure_key)
+
+ def index_item_location(item_location, current_start_date):
+ """ add this item to the search index """
+ item = self._fetch_item(modulestore, item_location)
+ if not item:
+ return
+
+ is_indexable = hasattr(item, "index_dictionary")
+ # if it's not indexable and it does not have children, then ignore
+ if not is_indexable and not item.has_children:
+ return
+
+ # if it has a defined start, then apply it and to it's children
+ if item.start and (not current_start_date or item.start > current_start_date):
+ current_start_date = item.start
+
+ if item.has_children:
+ for child_loc in item.children:
+ index_item_location(child_loc, current_start_date)
+
+ item_index = {}
+ item_index_dictionary = item.index_dictionary() if is_indexable else None
+
+ # if it has something to add to the index, then add it
+ if item_index_dictionary:
+ try:
+ item_index.update(location_info)
+ item_index.update(item_index_dictionary)
+ item_index['id'] = unicode(self._id_modifier(item.scope_ids.usage_id))
+ if current_start_date:
+ item_index['start_date'] = current_start_date
+
+ searcher.index(self.DOCUMENT_TYPE, item_index)
+ except Exception as err: # pylint: disable=broad-except
+ # broad exception so that index operation does not fail on one item of many
+ log.warning('Could not index item: %s - %s', item_location, unicode(err))
+ error_list.append(_('Could not index item: {}').format(item_location))
+
+ def remove_index_item_location(item_location):
+ """ remove this item from the search index """
+ item = self._fetch_item(modulestore, item_location)
+ if item:
+ if item.has_children:
+ for child_loc in item.children:
+ remove_index_item_location(child_loc)
+
+ target_location = item.scope_ids.usage_id
+ else:
+ target_location = item_location
+
+ searcher.remove(self.DOCUMENT_TYPE, unicode(self._id_modifier(target_location)))
+
+ try:
+ if delete:
+ remove_index_item_location(location)
+ else:
+ index_item_location(location, None)
+ indexed_count += 1
+ except Exception as err: # pylint: disable=broad-except
+ # broad exception so that index operation does not prevent the rest of the application from working
+ log.exception(
+ "Indexing error encountered, courseware index may be out of date %s - %s",
+ structure_key,
+ unicode(err)
+ )
+ error_list.append(_('General indexing error occurred'))
+
+ if raise_on_error and error_list:
+ raise SearchIndexingError(_('Error(s) present during indexing'), error_list)
+
+ return indexed_count
+
+
+class CoursewareSearchIndexer(SearchIndexerBase):
+ """
+ Class to perform indexing for courseware search from different modulestores
+ """
+ INDEX_NAME = "courseware_index"
+ DOCUMENT_TYPE = "courseware_content"
+
+ index_on_create = False
+ index_on_update = False
+
+ def _get_structure_key(self, location):
+ """ Gets structure key from location """
+ if isinstance(location, CourseLocator):
+ course_key = location
+ else:
+ course_key = location.course_key
+ return course_key
+
+ def _fetch_item(self, modulestore, item_location):
+ """ Fetch the item from the modulestore location, log if not found, but continue """
+ try:
+ if isinstance(item_location, CourseLocator):
+ item = modulestore.get_course(item_location)
+ else:
+ item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only)
+ except ItemNotFoundError:
+ log.warning('Cannot find: %s', item_location)
+ return None
+
+ return item
+
+ def do_course_reindex(self, modulestore, course_key):
+ """
+ (Re)index all content within the given course
+ """
+ indexed_count = self.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True)
+ self._track_index_request('edx.course.index.reindexed', indexed_count)
+ return indexed_count
+
+
+class LibrarySearchIndexer(SearchIndexerBase):
+ """
+ Class to perform indexing for library search from different modulestores
+ """
+ INDEX_NAME = "library_index"
+ DOCUMENT_TYPE = "library_content"
+
+ def _get_structure_key(self, location):
+ """ Gets structure key from location """
+ if isinstance(location, LibraryLocator):
+ course_key = location
+ else:
+ course_key = location.course_key.replace(version_guid=None, branch=None)
+ return course_key
+
+ def _get_location_info(self, structure_key):
+ """ Builds location info dictionary """
+ return {"library": unicode(structure_key)}
+
+ def _id_modifier(self, usage_id):
+ """ Modifies usage_id to submit to index """
+ return usage_id.replace(library_key=(usage_id.library_key.replace(version_guid=None, branch=None)))
+
+ def _fetch_item(self, modulestore, item_location):
+ """ Fetch the item from the modulestore location, log if not found, but continue """
+ try:
+ if isinstance(item_location, CourseLocator):
+ item = modulestore.get_library(item_location)
+ else:
+ item = modulestore.get_item(item_location)
+ except ItemNotFoundError:
+ log.warning('Cannot find: %s', item_location)
+ return None
+
+ return item
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
index 7d84e5612451..7bb99b6ef79e 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
@@ -5,7 +5,7 @@
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import ModuleStoreEnum
-from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
+from xmodule.modulestore.search_index import get_indexer_for_location
from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError
from xmodule.modulestore.draft_and_published import (
ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError
@@ -131,7 +131,12 @@ def update_item(self, descriptor, user_id, allow_not_found=False, force=False, *
)
self._auto_publish_no_children(item.location, item.location.category, user_id, **kwargs)
descriptor.location = old_descriptor_locn
- return item
+
+ indexer = get_indexer_for_location(item.location)
+ if indexer.index_on_update:
+ indexer.add_to_search_index(self, item.location)
+
+ return item
def create_item(
self, user_id, course_key, block_type, block_id=None,
@@ -152,7 +157,11 @@ def create_item(
)
if not skip_auto_publish:
self._auto_publish_no_children(item.location, item.location.category, user_id, **kwargs)
- return item
+
+ indexer = get_indexer_for_location(item.location)
+ if indexer.index_on_create:
+ indexer.add_to_search_index(self, item.location)
+ return item
def create_child(
self, user_id, parent_usage_key, block_type, block_id=None,
@@ -209,9 +218,11 @@ def delete_item(self, location, user_id, revision=None, **kwargs):
if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES:
self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
- # Remove this location from the courseware search index so that searches
+ # Remove this location from the search index so that searches
# will refrain from showing it as a result
- CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
+ indexer = get_indexer_for_location(location)
+ if indexer.index_on_delete:
+ indexer.add_to_search_index(self, location, delete=True)
def _map_revision_to_branch(self, key, revision=None):
"""
@@ -357,7 +368,9 @@ def publish(self, location, user_id, blacklist=None, **kwargs):
)
# Now it's been published, add the object to the courseware search index so that it appears in search results
- CoursewareSearchIndexer.do_publish_index(self, location)
+ indexer = get_indexer_for_location(location)
+ if indexer.index_on_publish:
+ indexer.do_publish_index(self, location)
return self.get_item(location.for_branch(ModuleStoreEnum.BranchName.published), **kwargs)
diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py b/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py
new file mode 100644
index 000000000000..db16ff98295b
--- /dev/null
+++ b/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py
@@ -0,0 +1,286 @@
+"""
+Unit tests for search indexers
+"""
+from django.conf import settings
+from django.test.utils import override_settings
+from lazy.lazy import lazy
+import os
+import mock
+import json
+
+from uuid import uuid4
+from datetime import datetime
+from django.test import TestCase
+
+from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator, LibraryUsageLocator
+
+from xmodule.modulestore import ModuleStoreWriteBase
+from xmodule.modulestore.search_index import get_indexer_for_location, CoursewareSearchIndexer, LibrarySearchIndexer
+
+
+class TestIndexerForLocation(TestCase):
+ """ Tests for search indexers factory method """
+ def test_given_course_locator_returns_courseware_indexer(self):
+ locator = mock.Mock(spec=CourseLocator)
+ indexer = get_indexer_for_location(locator)
+ self.assertIsInstance(indexer, CoursewareSearchIndexer)
+
+ def test_given_library_locator_returns_library_indexer(self):
+ locator = mock.Mock(spec=LibraryLocator)
+ indexer = get_indexer_for_location(locator)
+ self.assertIsInstance(indexer, LibrarySearchIndexer)
+
+ def test_given_course_item_locator_returns_courseware_indexer(self):
+ locator = mock.Mock(spec=BlockUsageLocator)
+ locator.course_key = mock.Mock(spec=CourseLocator)
+ indexer = get_indexer_for_location(locator)
+ self.assertIsInstance(indexer, CoursewareSearchIndexer)
+
+ def test_given_library_item_locator_returns_library_indexer(self):
+ locator = mock.Mock(spec=LibraryUsageLocator)
+ locator.course_key = mock.Mock(spec=LibraryLocator)
+ indexer = get_indexer_for_location(locator)
+ self.assertIsInstance(indexer, LibrarySearchIndexer)
+
+
+MOCK_SEARCH_ENGINE = getattr(settings, 'SEARCH_ENGINE', "search.tests.mock_search_engine.MockSearchEngine")
+
+
+# pylint: disable=no-member
+class SearchIndexerBaseTestMixin(object):
+ """ Common features for search indexers tests"""
+ ROOT_ID = None
+ modulestore = None
+
+ TEST_INDEX_FILENAME = getattr(
+ settings, 'MOCK_SEARCH_BACKING_FILE',
+ "test_root/index_file.dat"
+ )
+
+ def setUp(self):
+ """
+ Setting up mock search indexer backing file as prescribed by MOCK_SEARCH_BACKING_FILE setting
+ """
+ super(SearchIndexerBaseTestMixin, self).setUp()
+ # this test implicitly uses MockSearchEngine, which in turn uses settings.MOCK_SEARCH_BACKING_FILE
+ # as backing file - so we create it
+ with open(self.TEST_INDEX_FILENAME, "w+") as index_file:
+ json.dump({}, index_file)
+
+ self.modulestore = mock.Mock(spec=ModuleStoreWriteBase)
+
+ def tearDown(self):
+ """
+ Removing backing file created in set up
+ """
+ super(SearchIndexerBaseTestMixin, self).tearDown()
+ os.remove(self.TEST_INDEX_FILENAME)
+
+ def _make_location(self, block_type, course_id=ROOT_ID): # pylint: disable=unused-argument
+ """ Builds xblock location for specified block type and course id """
+ raise NotImplementedError()
+
+ def _process_index_id(self, location): # pylint: disable=unused-argument
+ """ Transforms block location to build correct index id """
+ raise NotImplementedError()
+
+ @lazy
+ def indexer(self):
+ """ Indexer under test instantiation """
+ raise NotImplementedError()
+
+ def _make_item(self, block_type, index_dictionary, course_id=None, start=None, children=None):
+ """ Builds XBlock mock with specified block_type, index disctionary, children, etc."""
+ location = self._make_location(block_type, course_id)
+ result = mock.Mock()
+ if index_dictionary is not None:
+ result.index_dictionary = mock.Mock(return_value=index_dictionary)
+ else:
+ del result.index_dictionary
+ result.start = start
+
+ result.children = children if children else []
+ result.has_children = len(result.children) > 0
+ result.location = location
+ result.scope_ids = mock.Mock()
+ result.scope_ids.usage_id = location
+ return result, location
+
+ def _set_up_modulestore_get_item(self, items):
+ """
+ Sets up mock modulestore get_item method to return only items listed in `items` parameter at specified locations
+ """
+ def side_effect(loc, **kwargs): # pylint: disable=unused-argument
+ """ Side-effect method for mock - gets item by location """
+ return items.get(loc, None)
+ self.modulestore.get_item = mock.Mock(side_effect=side_effect)
+
+ def _make_index_entry(self, index_dict, location, start_date=None):
+ """ Helper method: builds index entry dictionary """
+ result = {
+ 'id': self._process_index_id(location),
+ 'course': unicode(self.ROOT_ID),
+ }
+ result.update(index_dict)
+ if start_date:
+ result['start_date'] = start_date
+ return result
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.remove')
+ @mock.patch(MOCK_SEARCH_ENGINE + '.index')
+ @mock.patch('xmodule.modulestore.search_index.SearchEngine.get_search_engine')
+ def test_no_searcher_does_nothing(self, search_engine, index, remove):
+ """ Tests that does indexer does nothing if no search engine is available """
+ search_engine.return_value = None
+ _, location = self._make_item('html', {'item': 'value'})
+ self.indexer.add_to_search_index(self.modulestore, location)
+ search_engine.assert_called_with(self.indexer.INDEX_NAME)
+ index.assert_not_called()
+ remove.assert_not_called()
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.remove')
+ @mock.patch(MOCK_SEARCH_ENGINE + '.index')
+ def test_not_indexable_no_children_does_nothing(self, index, remove):
+ """ Tests that does indexer does nothing if xblock is not indexable """
+ item, location = self._make_item('html', None)
+ self._set_up_modulestore_get_item({location: item})
+ self.indexer.add_to_search_index(self.modulestore, location)
+ index.assert_not_called()
+ remove.assert_not_called()
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.index')
+ def test_add_to_index_no_children_adds_to_index(self, patched_index):
+ """ Tests that indexer adds XBlock with no children to index """
+ index_dict = {'item': 'value'}
+ item, location = self._make_item('html', index_dict)
+ self._set_up_modulestore_get_item({location: item})
+ self.indexer.add_to_search_index(self.modulestore, location)
+ expected_index_entry = self._make_index_entry(index_dict, location)
+ patched_index.assert_called_with(self.indexer.DOCUMENT_TYPE, expected_index_entry)
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.remove')
+ def test_add_to_index_with_delete_no_children_simple_removes_from_index(self, patched_remove):
+ """ Tests that indexer removes XBlock with no children from index """
+ item, location = self._make_item('html', {'item': 'value'})
+ self._set_up_modulestore_get_item({location: item})
+ self.indexer.add_to_search_index(self.modulestore, location, delete=True)
+ patched_remove.assert_called_with(self.indexer.DOCUMENT_TYPE, self._process_index_id(location))
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.index')
+ def test_add_to_index_with_chidlren_adds_all_to_index(self, patched_index):
+ """ Tests that indexer adds XBlock with children to index (both Xblock and all of its children)"""
+ index_dict_child1, index_dict_child2 = {'child': 'child1'}, {'child': 'child2'}
+ child1, child_loc1 = self._make_item('text', index_dict_child1)
+ child2, child_loc2 = self._make_item('html', index_dict_child2)
+
+ index_dict = {'item': 'value'}
+ start_date = datetime(2015, 7, 14, 22, 11, 03)
+
+ item, location = self._make_item(
+ 'problem', index_dict,
+ children=[child_loc1, child_loc2],
+ start=start_date
+ )
+ self._set_up_modulestore_get_item({location: item, child_loc1: child1, child_loc2: child2})
+ self.indexer.add_to_search_index(self.modulestore, location)
+
+ calls = patched_index.call_args_list
+ args, _ = calls[0]
+ self.assertEqual(args, (
+ self.indexer.DOCUMENT_TYPE,
+ self._make_index_entry(index_dict_child1, child_loc1, start_date=start_date)
+ ))
+ args, _ = calls[1]
+ self.assertEqual(args, (
+ self.indexer.DOCUMENT_TYPE,
+ self._make_index_entry(index_dict_child2, child_loc2, start_date=start_date)
+ ))
+ args, _ = calls[2]
+ self.assertEqual(args, (
+ self.indexer.DOCUMENT_TYPE,
+ self._make_index_entry(index_dict, location, start_date=start_date)
+ ))
+
+ @mock.patch(MOCK_SEARCH_ENGINE + '.remove')
+ def test_remove_from_index_with_chidlren_removes_all_from_index(self, patched_remove):
+ """ Tests that indexer removes XBlock with children from index (both Xblock and all of its children)"""
+ index_dict_child1, index_dict_child2 = {'child': 'child1'}, {'child': 'child2'}
+ child1, child_loc1 = self._make_item('text', index_dict_child1)
+ child2, child_loc2 = self._make_item('html', index_dict_child2)
+
+ index_dict = {'item': 'value'}
+
+ item, location = self._make_item('problem', index_dict, children=[child_loc1, child_loc2])
+ self._set_up_modulestore_get_item({location: item, child_loc1: child1, child_loc2: child2})
+ self.indexer.add_to_search_index(self.modulestore, location, delete=True)
+
+ calls = patched_remove.call_args_list
+ args, _ = calls[0]
+ self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(child_loc1)))
+ args, _ = calls[1]
+ self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(child_loc2)))
+ args, _ = calls[2]
+ self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(location)))
+
+
+@override_settings(MOCK_SEARCH_ENGINE=MOCK_SEARCH_ENGINE)
+class TestCoursewareSearchIndexer(SearchIndexerBaseTestMixin, TestCase):
+ """ Tests for CoursewareSearchIndexer """
+ ROOT_ID = CourseLocator('testx', 'courseware_indexer_test', 'test_run')
+
+ @lazy
+ def indexer(self):
+ """ Indexer under test instantiation """
+ return CoursewareSearchIndexer()
+
+ def _make_location(self, block_type, course_id=None):
+ """ Builds xblock location for specified block type and course id """
+ course_id = course_id if course_id else self.ROOT_ID
+ return BlockUsageLocator(course_id, block_type, uuid4().hex)
+
+ def _process_index_id(self, location):
+ """ Transforms block location to build correct index id """
+ return unicode(location)
+
+ def _make_index_entry(self, index_dict, location, start_date=None):
+ """ Helper method: builds index entry dictionary """
+ result = {
+ 'id': self._process_index_id(location),
+ 'course': unicode(self.ROOT_ID),
+ }
+ result.update(index_dict)
+ if start_date:
+ result['start_date'] = start_date
+ return result
+
+
+@override_settings(MOCK_SEARCH_ENGINE=MOCK_SEARCH_ENGINE)
+class TestLibrarySearchIndexer(SearchIndexerBaseTestMixin, TestCase):
+ """ Tests for LibrarySearchIndexer """
+ ROOT_ID = LibraryLocator('lib', 'Lib1')
+
+ @lazy
+ def indexer(self):
+ """ Indexer under test instantiation """
+ return LibrarySearchIndexer()
+
+ def _make_location(self, block_type, course_id=None):
+ """ Builds xblock location for specified block type and course id """
+ course_id = course_id if course_id else self.ROOT_ID
+ return LibraryUsageLocator(course_id, block_type, uuid4().hex)
+
+ def _process_index_id(self, location):
+ """ Transforms block location to build correct index id """
+ new_loc = location.replace(library_key=location.library_key.replace(version_guid=None, branch=None))
+ return unicode(new_loc)
+
+ def _make_index_entry(self, index_dict, location, start_date=None):
+ """ Helper method: builds index entry dictionary """
+ result = {
+ 'id': self._process_index_id(location),
+ 'library': unicode(self.ROOT_ID),
+ }
+ result.update(index_dict)
+ if start_date:
+ result['start_date'] = start_date
+ return result
diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py
index d1309fded457..3a58a118d6f4 100644
--- a/common/lib/xmodule/xmodule/tests/test_capa_module.py
+++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py
@@ -1666,18 +1666,26 @@ def test_check_unmask_answerpool(self):
@ddt.ddt
class CapaDescriptorTest(unittest.TestCase):
- def _create_descriptor(self, xml):
+ def _create_descriptor(self, xml, name=None):
""" Creates a CapaDescriptor to run test against """
descriptor = CapaDescriptor(get_test_system(), scope_ids=1)
descriptor.data = xml
+ if name:
+ descriptor.display_name = name
return descriptor
@ddt.data(*responsetypes.registry.registered_tags())
def test_all_response_types(self, response_tag):
""" Tests that every registered response tag is correctly returned """
xml = "<{response_tag}>{response_tag}>".format(response_tag=response_tag)
- descriptor = self._create_descriptor(xml)
+ name = "Some Capa Problem"
+ descriptor = self._create_descriptor(xml, name=name)
self.assertEquals(descriptor.problem_types, {response_tag})
+ self.assertEquals(descriptor.index_dictionary(), {
+ 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE,
+ 'display_name': name,
+ 'problem_types': [response_tag]
+ })
def test_response_types_ignores_non_response_tags(self):
xml = textwrap.dedent("""
@@ -1694,8 +1702,14 @@ def test_response_types_ignores_non_response_tags(self):
""")
- descriptor = self._create_descriptor(xml)
+ name = "Test Capa Problem"
+ descriptor = self._create_descriptor(xml, name=name)
self.assertEquals(descriptor.problem_types, {"multiplechoiceresponse"})
+ self.assertEquals(descriptor.index_dictionary(), {
+ 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE,
+ 'display_name': name,
+ 'problem_types': ["multiplechoiceresponse"]
+ })
def test_response_types_multiple_tags(self):
xml = textwrap.dedent("""
@@ -1717,8 +1731,16 @@ def test_response_types_multiple_tags(self):
""")
- descriptor = self._create_descriptor(xml)
+ name = "Other Test Capa Problem"
+ descriptor = self._create_descriptor(xml, name=name)
self.assertEquals(descriptor.problem_types, {"multiplechoiceresponse", "optionresponse"})
+ self.assertEquals(
+ descriptor.index_dictionary(), {
+ 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE,
+ 'display_name': name,
+ 'problem_types': ["optionresponse", "multiplechoiceresponse"]
+ }
+ )
class ComplexEncoderTest(unittest.TestCase):
diff --git a/common/lib/xmodule/xmodule/tests/test_library_content.py b/common/lib/xmodule/xmodule/tests/test_library_content.py
index fd9313f8463c..cbe9ad301867 100644
--- a/common/lib/xmodule/xmodule/tests/test_library_content.py
+++ b/common/lib/xmodule/xmodule/tests/test_library_content.py
@@ -18,6 +18,7 @@
from xmodule.tests import get_test_system
from xmodule.validation import StudioValidationMessage
from xmodule.x_module import AUTHOR_VIEW
+from search.search_engine_base import SearchEngine
dummy_render = lambda block, _: Fragment(block.data) # pylint: disable=invalid-name
@@ -66,10 +67,17 @@ def get_module(descriptor):
module.xmodule_runtime = module_system
-class TestLibraryContentModule(LibraryContentTest):
+class LibraryContentModuleTestMixin(object):
"""
Basic unit tests for LibraryContentModule
"""
+ problem_types = [
+ ["multiplechoiceresponse"], ["optionresponse"], ["optionresponse", "coderesponse"],
+ ["coderesponse", "optionresponse"]
+ ]
+
+ problem_type_lookup = {}
+
def _get_capa_problem_type_xml(self, *args):
""" Helper function to create empty CAPA problem definition """
problem = ""
@@ -84,12 +92,10 @@ def _create_capa_problems(self):
Creates four blocks total.
"""
- problem_types = [
- ["multiplechoiceresponse"], ["optionresponse"], ["optionresponse", "coderesponse"],
- ["coderesponse", "optionresponse"]
- ]
- for problem_type in problem_types:
- self.make_block("problem", self.library, data=self._get_capa_problem_type_xml(*problem_type))
+ self.problem_type_lookup = {}
+ for problem_type in self.problem_types:
+ block = self.make_block("problem", self.library, data=self._get_capa_problem_type_xml(*problem_type))
+ self.problem_type_lookup[block.location] = problem_type
def test_lib_content_block(self):
"""
@@ -236,6 +242,42 @@ def test_non_editable_settings(self):
self.assertNotIn(LibraryContentDescriptor.display_name, non_editable_metadata_fields)
+@patch('xmodule.library_tools.SearchEngine.get_search_engine', Mock(return_value=None))
+class TestLibraryContentModuleNoSearchIndex(LibraryContentModuleTestMixin, LibraryContentTest):
+ """
+ Tests for library container when no search index is available.
+ Tests fallback low-level CAPA problem introspection
+ """
+ pass
+
+
+search_index_mock = Mock(spec=SearchEngine) # pylint: disable=invalid-name
+
+
+@patch('xmodule.library_tools.SearchEngine.get_search_engine', Mock(return_value=search_index_mock))
+class TestLibraryContentModuleWithSearchIndex(LibraryContentModuleTestMixin, LibraryContentTest):
+ """
+ Tests for library container with mocked search engine response.
+ """
+ def _get_search_response(self, field_dictionary=None):
+ """ Mocks search response as returned by search engine """
+ target_type = field_dictionary.get('problem_types')
+ matched_block_locations = [
+ key for key, problem_types in
+ self.problem_type_lookup.items() if target_type in problem_types
+ ]
+ return {
+ 'results': [
+ {'data': {'id': str(location)}} for location in matched_block_locations
+ ]
+ }
+
+ def setUp(self):
+ """ Sets up search engine mock """
+ super(TestLibraryContentModuleWithSearchIndex, self).setUp()
+ search_index_mock.search = Mock(side_effect=self._get_search_response)
+
+
@patch(
'xmodule.modulestore.split_mongo.caching_descriptor_system.CachingDescriptorSystem.render', VanillaRuntime.render
)
diff --git a/common/test/acceptance/tests/helpers.py b/common/test/acceptance/tests/helpers.py
index 48204fb575cf..fcf3eefe8fab 100644
--- a/common/test/acceptance/tests/helpers.py
+++ b/common/test/acceptance/tests/helpers.py
@@ -417,3 +417,17 @@ def create_user_partition_json(partition_id, name, description, groups, scheme="
return UserPartition(
partition_id, name, description, groups, MockUserPartitionScheme(scheme)
).to_json()
+
+
+class TestWithSearchIndexMixin(object):
+ """ Mixin encapsulating search index creation """
+ TEST_INDEX_FILENAME = "test_root/index_file.dat"
+
+ def _create_search_index(self):
+ """ Creates search index backing file """
+ with open(self.TEST_INDEX_FILENAME, "w+") as index_file:
+ json.dump({}, index_file)
+
+ def _cleanup_index_file(self):
+ """ Removes search index backing file """
+ os.remove(self.TEST_INDEX_FILENAME)
diff --git a/common/test/acceptance/tests/lms/test_library.py b/common/test/acceptance/tests/lms/test_library.py
index e4bb1cd8ec09..143acc77f685 100644
--- a/common/test/acceptance/tests/lms/test_library.py
+++ b/common/test/acceptance/tests/lms/test_library.py
@@ -6,7 +6,7 @@
import textwrap
from nose.plugins.attrib import attr
-from ..helpers import UniqueCourseTest
+from ..helpers import UniqueCourseTest, TestWithSearchIndexMixin
from ...pages.studio.auto_auth import AutoAuthPage
from ...pages.studio.overview import CourseOutlinePage
from ...pages.studio.library import StudioLibraryContentEditor, StudioLibraryContainerXBlockWrapper
@@ -196,10 +196,19 @@ def test_shows_all_if_max_set_to_greater_value(self):
@ddt.ddt
@attr('shard_3')
-class StudioLibraryContainerCapaFilterTest(LibraryContentTestBase):
+class StudioLibraryContainerCapaFilterTest(LibraryContentTestBase, TestWithSearchIndexMixin):
"""
Test Library Content block in LMS
"""
+ def setUp(self):
+ """ SetUp method """
+ self._create_search_index()
+ super(StudioLibraryContainerCapaFilterTest, self).setUp()
+
+ def tearDown(self):
+ self._cleanup_index_file()
+ super(StudioLibraryContainerCapaFilterTest, self).tearDown()
+
def _get_problem_choice_group_text(self, name, items):
""" Generates Choice Group CAPA problem XML """
items_text = "\n".join([
@@ -231,7 +240,7 @@ def populate_library_fixture(self, library_fixture):
"""
Populates library fixture with XBlock Fixtures
"""
- library_fixture.add_children(
+ items = (
XBlockFixtureDesc(
"problem", "Problem Choice Group 1",
data=self._get_problem_choice_group_text("Problem Choice Group 1 Text", [("1", False), ('2', True)])
@@ -249,6 +258,7 @@ def populate_library_fixture(self, library_fixture):
data=self._get_problem_select_text("Problem Select 2 Text", ["Option 3", "Option 4"], "Option 4")
),
)
+ library_fixture.add_children(*items)
@property
def _problem_headers(self):
diff --git a/common/test/acceptance/tests/studio/test_studio_library_container.py b/common/test/acceptance/tests/studio/test_studio_library_container.py
index 772b9013a001..20a4f111b67f 100644
--- a/common/test/acceptance/tests/studio/test_studio_library_container.py
+++ b/common/test/acceptance/tests/studio/test_studio_library_container.py
@@ -7,7 +7,7 @@
from .base_studio_test import StudioLibraryTest
from ...fixtures.course import CourseFixture
-from ..helpers import UniqueCourseTest
+from ..helpers import UniqueCourseTest, TestWithSearchIndexMixin
from ...pages.studio.library import StudioLibraryContentEditor, StudioLibraryContainerXBlockWrapper
from ...pages.studio.overview import CourseOutlinePage
from ...fixtures.course import XBlockFixtureDesc
@@ -18,7 +18,7 @@
@ddt.ddt
-class StudioLibraryContainerTest(StudioLibraryTest, UniqueCourseTest):
+class StudioLibraryContainerTest(StudioLibraryTest, UniqueCourseTest, TestWithSearchIndexMixin):
"""
Test Library Content block in LMS
"""
@@ -26,6 +26,7 @@ def setUp(self):
"""
Install library with some content and a course using fixtures
"""
+ self._create_search_index()
super(StudioLibraryContainerTest, self).setUp()
# Also create a course:
self.course_fixture = CourseFixture(
@@ -42,6 +43,11 @@ def setUp(self):
subsection = self.outline.section(SECTION_NAME).subsection(SUBSECTION_NAME)
self.unit_page = subsection.expand_subsection().unit(UNIT_NAME).go_to()
+ def tearDown(self):
+ """ Cleans up search index backing file """
+ self._cleanup_index_file()
+ super(StudioLibraryContainerTest, self).tearDown()
+
def populate_library_fixture(self, library_fixture):
"""
Populate the children of the test course fixture.
diff --git a/lms/envs/test.py b/lms/envs/test.py
index 420449698893..6af25730528a 100644
--- a/lms/envs/test.py
+++ b/lms/envs/test.py
@@ -462,6 +462,9 @@
FEATURES['ENABLE_COURSEWARE_SEARCH'] = True
# Use MockSearchEngine as the search engine for test scenario
SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
+MOCK_SEARCH_BACKING_FILE = (
+ TEST_ROOT / "index_file.dat" # pylint: disable=no-value-for-parameter
+).abspath()
FACEBOOK_APP_SECRET = "Test"
FACEBOOK_APP_ID = "Test"
diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt
index 2c3c0ac1766f..e7c8f9d46ebf 100644
--- a/requirements/edx/github.txt
+++ b/requirements/edx/github.txt
@@ -38,7 +38,7 @@ git+https://github.com/mitocw/django-cas.git@60a5b8e5a62e63e0d5d224a87f0b489201a
-e git+https://github.com/edx/edx-val.git@64aa7637e3459fb3000a85a9e156880a40307dd1#egg=edx-val
-e git+https://github.com/pmitros/RecommenderXBlock.git@9b07e807c89ba5761827d0387177f71aa57ef056#egg=recommender-xblock
-e git+https://github.com/edx/edx-milestones.git@547f2250ee49e73ce8d7ff4e78ecf1b049892510#egg=edx-milestones
--e git+https://github.com/edx/edx-search.git@21ac6b06b3bfe789dcaeaf4e2ab5b00a688324d4#egg=edx-search
+-e git+https://github.com/edx/edx-search.git@b5444b2702817bc04f5d0d025779ee50dbdec465#egg=edx-search
git+https://github.com/edx/edx-lint.git@8bf82a32ecb8598c415413df66f5232ab8d974e9#egg=edx_lint==0.2.1
-e git+https://github.com/edx/xblock-utils.git@17e247d66fabb53f0453515b093a030ee345a1b0#egg=xblock-utils
-e git+https://github.com/edx-solutions/xblock-google-drive.git@138e6fa0bf3a2013e904a085b9fed77dab7f3f21#egg=xblock-google-drive