diff --git a/cms/djangoapps/contentstore/tests/test_libraries.py b/cms/djangoapps/contentstore/tests/test_libraries.py index 0ff0bfabce16..cdc8d051dca0 100644 --- a/cms/djangoapps/contentstore/tests/test_libraries.py +++ b/cms/djangoapps/contentstore/tests/test_libraries.py @@ -385,6 +385,7 @@ def test_refreshes_children_if_libraries_change(self): html_block = modulestore().get_item(lc_block.children[0]) self.assertEqual(html_block.data, data2) + @patch("xmodule.library_tools.SearchEngine.get_search_engine", Mock(return_value=None)) def test_refreshes_children_if_capa_type_change(self): """ Tests that children are automatically refreshed if capa type field changes """ name1, name2 = "Option Problem", "Multiple Choice Problem" diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 7db80dc44f6d..cafd74d09006 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -22,7 +22,7 @@ from xmodule.course_module import DEFAULT_START_DATE from xmodule.error_module import ErrorDescriptor from xmodule.modulestore.django import modulestore -from xmodule.modulestore.courseware_index import CoursewareSearchIndexer, SearchIndexingError +from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError from xmodule.contentstore.content import StaticContent from xmodule.tabs import PDFTextbookTabs from xmodule.partitions.partitions import UserPartition @@ -137,7 +137,8 @@ def reindex_course_and_check_access(course_key, user): """ if not has_course_author_access(user, course_key): raise PermissionDenied() - return CoursewareSearchIndexer.do_course_reindex(modulestore(), course_key) + indexer = get_indexer_for_location(course_key) + return indexer.do_course_reindex(modulestore(), course_key) @login_required diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index 6c110839663a..9d154f41cef9 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -15,11 +15,10 @@ from course_action_state.models import CourseRerunState from util.date_utils import get_default_time_display from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.courseware_index import CoursewareSearchIndexer +from xmodule.modulestore.search_index import get_indexer_for_location, SearchIndexingError from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory -from xmodule.modulestore.courseware_index import SearchIndexingError from opaque_keys.edx.locator import CourseLocator from student.tests.factories import UserFactory from course_action_state.managers import CourseRerunUIStateManager @@ -385,6 +384,20 @@ def setUp(self): self.addCleanup(os.remove, self.TEST_INDEX_FILENAME) + def _perform_search(self, query="unique"): + """ Performs search """ + return perform_search( + query, + user=self.user, # pylint: disable=no-member + size=10, + from_=0, + course_id=unicode(self.course.id)) + + def _do_reindex(self): + """ Reindexes course """ + indexer = get_indexer_for_location(self.course.id) + return indexer.do_course_reindex(modulestore(), self.course.id) + def test_reindex_course(self): """ Verify that course gets reindexed. @@ -446,12 +459,7 @@ def test_reindex_json_responses(self): Test json response with real data """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # Start manual reindex @@ -464,12 +472,7 @@ def test_reindex_json_responses(self): reindex_course_and_check_access(self.course.id, self.user) # Check results indexed now - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['total'], 1) @mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary') @@ -478,12 +481,7 @@ def test_reindex_video_error_json_responses(self, mock_index_dictionary): Test json response with mocked error data for video """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response @@ -500,12 +498,7 @@ def test_reindex_html_error_json_responses(self, mock_index_dictionary): Test json response with mocked error data for html """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response @@ -522,12 +515,7 @@ def test_reindex_seq_error_json_responses(self, mock_index_dictionary): Test json response with mocked error data for sequence """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response @@ -562,30 +550,20 @@ def test_indexing_responses(self): Test add_to_search_index response with real data """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # Start manual reindex - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() self.html.display_name = "My expanded HTML" modulestore().update_item(self.html, ModuleStoreEnum.UserID.test) # Start manual reindex - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() # Check results indexed now - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['total'], 1) @mock.patch('xmodule.video_module.VideoDescriptor.index_dictionary') @@ -594,21 +572,15 @@ def test_indexing_video_error_responses(self, mock_index_dictionary): Test add_to_search_index response with mocked error data for video """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response - err = Exception - mock_index_dictionary.return_value = err + mock_index_dictionary.return_value = Exception # Start manual reindex and check error in response with self.assertRaises(SearchIndexingError): - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() @mock.patch('xmodule.html_module.HtmlDescriptor.index_dictionary') def test_indexing_html_error_responses(self, mock_index_dictionary): @@ -616,21 +588,15 @@ def test_indexing_html_error_responses(self, mock_index_dictionary): Test add_to_search_index response with mocked error data for html """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response - err = Exception - mock_index_dictionary.return_value = err + mock_index_dictionary.return_value = Exception # Start manual reindex and check error in response with self.assertRaises(SearchIndexingError): - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() @mock.patch('xmodule.seq_module.SequenceDescriptor.index_dictionary') def test_indexing_seq_error_responses(self, mock_index_dictionary): @@ -638,21 +604,15 @@ def test_indexing_seq_error_responses(self, mock_index_dictionary): Test add_to_search_index response with mocked error data for sequence """ # Check results not indexed - response = perform_search( - "unique", - user=self.user, - size=10, - from_=0, - course_id=unicode(self.course.id)) + response = self._perform_search() self.assertEqual(response['results'], []) # set mocked exception response - err = Exception - mock_index_dictionary.return_value = err + mock_index_dictionary.return_value = Exception # Start manual reindex and check error in response with self.assertRaises(SearchIndexingError): - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() @mock.patch('xmodule.modulestore.mongo.base.MongoModuleStore.get_course') def test_indexing_no_item(self, mock_get_course): @@ -660,9 +620,8 @@ def test_indexing_no_item(self, mock_get_course): Test system logs an error if no item found. """ # set mocked exception response - err = ItemNotFoundError - mock_get_course.return_value = err + mock_get_course.return_value = ItemNotFoundError # Start manual reindex and check error in response with self.assertRaises(SearchIndexingError): - CoursewareSearchIndexer.do_course_reindex(modulestore(), self.course.id) + self._do_reindex() diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 7364730815ec..d647aa8c0ddf 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -113,6 +113,7 @@ class CapaDescriptor(CapaFields, RawDescriptor): Module implementing problems in the LON-CAPA format, as implemented by capa.capa_problem """ + INDEX_CONTENT_TYPE = 'CAPA' module_class = CapaModule @@ -186,6 +187,21 @@ def problem_types(self): registered_tags = responsetypes.registry.registered_tags() return set([node.tag for node in tree.iter() if node.tag in registered_tags]) + def index_dictionary(self): + """ + Return dictionary prepared with module content and type for indexing. + """ + result = super(CapaDescriptor, self).index_dictionary() + if not result: + result = {} + index = { + 'content_type': self.INDEX_CONTENT_TYPE, + 'problem_types': list(self.problem_types), + "display_name": self.display_name + } + result.update(index) + return result + # Proxy to CapaModule for access to any of its attributes answer_available = module_attr('answer_available') check_button_name = module_attr('check_button_name') diff --git a/common/lib/xmodule/xmodule/library_tools.py b/common/lib/xmodule/xmodule/library_tools.py index 40afbbb549e1..ee657f619f02 100644 --- a/common/lib/xmodule/xmodule/library_tools.py +++ b/common/lib/xmodule/xmodule/library_tools.py @@ -2,7 +2,8 @@ XBlock runtime services for LibraryContentModule """ from django.core.exceptions import PermissionDenied -from opaque_keys.edx.locator import LibraryLocator +from opaque_keys.edx.locator import LibraryLocator, LibraryUsageLocator +from search.search_engine_base import SearchEngine from xmodule.library_content_module import ANY_CAPA_TYPE_VALUE from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.capa_module import CapaDescriptor @@ -82,13 +83,24 @@ def summarize_block(usage_key): result_json.append(info) return result_json + def _problem_type_filter(self, library, capa_type): + """ Filters library children by capa type""" + search_engine = SearchEngine.get_search_engine(index="library_index") + if search_engine: + filter_clause = { + "content_type": CapaDescriptor.INDEX_CONTENT_TYPE, + "problem_types": capa_type + } + search_result = search_engine.search(field_dictionary=filter_clause) + results = search_result.get('results', []) + return [LibraryUsageLocator.from_string(item['data']['id']) for item in results] + else: + return [key for key in library.children if self._filter_child(key, capa_type)] + def _filter_child(self, usage_key, capa_type): """ Filters children by CAPA problem type, if configured """ - if capa_type == ANY_CAPA_TYPE_VALUE: - return True - if usage_key.block_type != "problem": return False @@ -131,7 +143,7 @@ def update_children(self, dest_block, user_id, user_perms=None): filter_children = (dest_block.capa_type != ANY_CAPA_TYPE_VALUE) if filter_children: # Apply simple filtering based on CAPA problem types: - source_blocks.extend([key for key in library.children if self._filter_child(key, dest_block.capa_type)]) + source_blocks.extend(self._problem_type_filter(library, dest_block.capa_type)) else: source_blocks.extend(library.children) diff --git a/common/lib/xmodule/xmodule/modulestore/courseware_index.py b/common/lib/xmodule/xmodule/modulestore/courseware_index.py deleted file mode 100644 index ff55b1a721e2..000000000000 --- a/common/lib/xmodule/xmodule/modulestore/courseware_index.py +++ /dev/null @@ -1,176 +0,0 @@ -""" Code to allow module store to interface with courseware index """ -from __future__ import absolute_import - -import logging - -from django.utils.translation import ugettext as _ -from opaque_keys.edx.locator import CourseLocator -from search.search_engine_base import SearchEngine -from eventtracking import tracker - -from . import ModuleStoreEnum -from .exceptions import ItemNotFoundError - - -# Use default index and document names for now -INDEX_NAME = "courseware_index" -DOCUMENT_TYPE = "courseware_content" - -log = logging.getLogger('edx.modulestore') - - -class SearchIndexingError(Exception): - """ Indicates some error(s) occured during indexing """ - - def __init__(self, message, error_list): - super(SearchIndexingError, self).__init__(message) - self.error_list = error_list - - -class CoursewareSearchIndexer(object): - """ - Class to perform indexing for courseware search from different modulestores - """ - - @staticmethod - def add_to_search_index(modulestore, location, delete=False, raise_on_error=False): - """ - Add to courseware search index from given location and its children - """ - error_list = [] - indexed_count = 0 - # TODO - inline for now, need to move this out to a celery task - searcher = SearchEngine.get_search_engine(INDEX_NAME) - if not searcher: - return - - if isinstance(location, CourseLocator): - course_key = location - else: - course_key = location.course_key - - location_info = { - "course": unicode(course_key), - } - - def _fetch_item(item_location): - """ Fetch the item from the modulestore location, log if not found, but continue """ - try: - if isinstance(item_location, CourseLocator): - item = modulestore.get_course(item_location) - else: - item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only) - except ItemNotFoundError: - log.warning('Cannot find: %s', item_location) - return None - - return item - - def index_item_location(item_location, current_start_date): - """ add this item to the search index """ - item = _fetch_item(item_location) - if not item: - return - - is_indexable = hasattr(item, "index_dictionary") - # if it's not indexable and it does not have children, then ignore - if not is_indexable and not item.has_children: - return - - # if it has a defined start, then apply it and to it's children - if item.start and (not current_start_date or item.start > current_start_date): - current_start_date = item.start - - if item.has_children: - for child_loc in item.children: - index_item_location(child_loc, current_start_date) - - item_index = {} - item_index_dictionary = item.index_dictionary() if is_indexable else None - - # if it has something to add to the index, then add it - if item_index_dictionary: - try: - item_index.update(location_info) - item_index.update(item_index_dictionary) - item_index['id'] = unicode(item.scope_ids.usage_id) - if current_start_date: - item_index['start_date'] = current_start_date - - searcher.index(DOCUMENT_TYPE, item_index) - except Exception as err: # pylint: disable=broad-except - # broad exception so that index operation does not fail on one item of many - log.warning('Could not index item: %s - %s', item_location, unicode(err)) - error_list.append(_('Could not index item: {}').format(item_location)) - - def remove_index_item_location(item_location): - """ remove this item from the search index """ - item = _fetch_item(item_location) - if item: - if item.has_children: - for child_loc in item.children: - remove_index_item_location(child_loc) - - searcher.remove(DOCUMENT_TYPE, unicode(item.scope_ids.usage_id)) - - try: - if delete: - remove_index_item_location(location) - else: - index_item_location(location, None) - indexed_count += 1 - except Exception as err: # pylint: disable=broad-except - # broad exception so that index operation does not prevent the rest of the application from working - log.exception( - "Indexing error encountered, courseware index may be out of date %s - %s", - course_key, - unicode(err) - ) - error_list.append(_('General indexing error occurred')) - - if raise_on_error and error_list: - raise SearchIndexingError(_('Error(s) present during indexing'), error_list) - - return indexed_count - - @classmethod - def do_publish_index(cls, modulestore, location, delete=False, raise_on_error=False): - """ - Add to courseware search index published section and children - """ - indexed_count = cls.add_to_search_index(modulestore, location, delete, raise_on_error) - cls._track_index_request('edx.course.index.published', indexed_count, str(location)) - return indexed_count - - @classmethod - def do_course_reindex(cls, modulestore, course_key): - """ - (Re)index all content within the given course - """ - indexed_count = cls.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True) - cls._track_index_request('edx.course.index.reindexed', indexed_count) - return indexed_count - - @staticmethod - def _track_index_request(event_name, indexed_count, location=None): - """Track content index requests. - - Arguments: - location (str): The ID of content to be indexed. - event_name (str): Name of the event to be logged. - Returns: - None - - """ - data = { - "indexed_count": indexed_count, - 'category': 'courseware_index', - } - - if location: - data['location_id'] = location - - tracker.emit( - event_name, - data - ) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py index 5c30f5e379e3..731b3d235386 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/draft.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/draft.py @@ -12,7 +12,7 @@ from opaque_keys.edx.locations import Location from xmodule.exceptions import InvalidVersionError from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.courseware_index import CoursewareSearchIndexer +from xmodule.modulestore.search_index import get_indexer_for_location from xmodule.modulestore.exceptions import ( ItemNotFoundError, DuplicateItemError, DuplicateCourseError, InvalidBranchSetting ) @@ -469,6 +469,11 @@ def update_item(self, xblock, user_id, allow_not_found=False, force=False, isPub xblock.location = draft_loc super(DraftModuleStore, self).update_item(xblock, user_id, allow_not_found, isPublish=isPublish) + + indexer = get_indexer_for_location(xblock.location) + if indexer.index_on_update: + indexer.add_to_search_index(self, xblock.location) + return wrap_draft(xblock) def delete_item(self, location, user_id, revision=None, **kwargs): @@ -552,7 +557,9 @@ def delete_item(self, location, user_id, revision=None, **kwargs): # Remove this location from the courseware search index so that searches # will refrain from showing it as a result - CoursewareSearchIndexer.add_to_search_index(self, location, delete=True) + indexer = get_indexer_for_location(location) + if indexer.index_on_delete: + indexer.add_to_search_index(self, location, delete=True) def _delete_subtree(self, location, as_functions, draft_only=False): """ @@ -732,7 +739,9 @@ def _internal_depth_first(item_location, is_root): self.signal_handler.send("course_published", course_key=course_key) # Now it's been published, add the object to the courseware search index so that it appears in search results - CoursewareSearchIndexer.do_publish_index(self, location) + indexer = get_indexer_for_location(location) + if indexer.index_on_publish: + indexer.do_publish_index(self, location) return self.get_item(as_published(location)) diff --git a/common/lib/xmodule/xmodule/modulestore/search_index.py b/common/lib/xmodule/xmodule/modulestore/search_index.py new file mode 100644 index 000000000000..5852a521093b --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/search_index.py @@ -0,0 +1,264 @@ +""" Code to allow module store to interface with courseware index """ +from __future__ import absolute_import + +import logging + +from django.utils.translation import ugettext as _ +from opaque_keys.edx.locator import CourseLocator, LibraryLocator +from search.search_engine_base import SearchEngine +from eventtracking import tracker + +from . import ModuleStoreEnum +from .exceptions import ItemNotFoundError + + +# Use default index and document names for now + + +log = logging.getLogger('edx.modulestore') + + +def get_indexer_for_location(locator): + """ + Initializes correct indexer for given location. + + Arguments: + locator BlockLocatorBase: XBlock locator + """ + if isinstance(locator, CourseLocator): + return CoursewareSearchIndexer() + elif isinstance(locator, LibraryLocator): + return LibrarySearchIndexer() + return get_indexer_for_location(locator.course_key) + + +class SearchIndexingError(Exception): + """ Indicates some error(s) occured during indexing """ + + def __init__(self, message, error_list): + super(SearchIndexingError, self).__init__(message) + self.error_list = error_list + + +class SearchIndexerBase(object): + """ + Base class to perform XBlock indexing from different modulestores + """ + INDEX_NAME = None + DOCUMENT_TYPE = None + + index_on_create = True + index_on_update = True + index_on_delete = True + index_on_publish = True + + def _get_structure_key(self, location): + """ Gets structure key from location """ + return location.course_key + + def _get_location_info(self, structure_key): + """ Builds location info dictionary """ + return {"course": unicode(structure_key)} + + def _fetch_item(self, modulestore, item_location): # pylint: disable=unused-argument + """ Fetch the item from the modulestore location, log if not found, but continue """ + raise NotImplementedError() + + def _id_modifier(self, usage_id): + """ Modifies usage_id to submit to index """ + return usage_id + + @classmethod + def _track_index_request(cls, event_name, indexed_count, location=None): + """Track content index requests. + + def add_to_search_index(self, modulestore, location, delete=False, raise_on_error=False): + Arguments: + location (str): The ID of content to be indexed. + event_name (str): Name of the event to be logged. + Returns: + None + + """ + data = { + "indexed_count": indexed_count, + 'category': cls.INDEX_NAME, + } + + if location: + data['location_id'] = location + + tracker.emit( + event_name, + data + ) + + def do_publish_index(self, modulestore, location, delete=False, raise_on_error=False): + """ + Add to search index published section and children + """ + indexed_count = self.add_to_search_index(modulestore, location, delete, raise_on_error) + self._track_index_request('edx.course.index.published', indexed_count, str(location)) + return indexed_count + + def add_to_search_index(self, modulestore, location, delete=False, raise_on_error=False): + """ + Add to courseware search index from given location and its children + """ + error_list = [] + indexed_count = 0 + # TODO - inline for now, need to move this out to a celery task + searcher = SearchEngine.get_search_engine(self.INDEX_NAME) + if not searcher: + return + + structure_key = self._get_structure_key(location) + location_info = self._get_location_info(structure_key) + + def index_item_location(item_location, current_start_date): + """ add this item to the search index """ + item = self._fetch_item(modulestore, item_location) + if not item: + return + + is_indexable = hasattr(item, "index_dictionary") + # if it's not indexable and it does not have children, then ignore + if not is_indexable and not item.has_children: + return + + # if it has a defined start, then apply it and to it's children + if item.start and (not current_start_date or item.start > current_start_date): + current_start_date = item.start + + if item.has_children: + for child_loc in item.children: + index_item_location(child_loc, current_start_date) + + item_index = {} + item_index_dictionary = item.index_dictionary() if is_indexable else None + + # if it has something to add to the index, then add it + if item_index_dictionary: + try: + item_index.update(location_info) + item_index.update(item_index_dictionary) + item_index['id'] = unicode(self._id_modifier(item.scope_ids.usage_id)) + if current_start_date: + item_index['start_date'] = current_start_date + + searcher.index(self.DOCUMENT_TYPE, item_index) + except Exception as err: # pylint: disable=broad-except + # broad exception so that index operation does not fail on one item of many + log.warning('Could not index item: %s - %s', item_location, unicode(err)) + error_list.append(_('Could not index item: {}').format(item_location)) + + def remove_index_item_location(item_location): + """ remove this item from the search index """ + item = self._fetch_item(modulestore, item_location) + if item: + if item.has_children: + for child_loc in item.children: + remove_index_item_location(child_loc) + + target_location = item.scope_ids.usage_id + else: + target_location = item_location + + searcher.remove(self.DOCUMENT_TYPE, unicode(self._id_modifier(target_location))) + + try: + if delete: + remove_index_item_location(location) + else: + index_item_location(location, None) + indexed_count += 1 + except Exception as err: # pylint: disable=broad-except + # broad exception so that index operation does not prevent the rest of the application from working + log.exception( + "Indexing error encountered, courseware index may be out of date %s - %s", + structure_key, + unicode(err) + ) + error_list.append(_('General indexing error occurred')) + + if raise_on_error and error_list: + raise SearchIndexingError(_('Error(s) present during indexing'), error_list) + + return indexed_count + + +class CoursewareSearchIndexer(SearchIndexerBase): + """ + Class to perform indexing for courseware search from different modulestores + """ + INDEX_NAME = "courseware_index" + DOCUMENT_TYPE = "courseware_content" + + index_on_create = False + index_on_update = False + + def _get_structure_key(self, location): + """ Gets structure key from location """ + if isinstance(location, CourseLocator): + course_key = location + else: + course_key = location.course_key + return course_key + + def _fetch_item(self, modulestore, item_location): + """ Fetch the item from the modulestore location, log if not found, but continue """ + try: + if isinstance(item_location, CourseLocator): + item = modulestore.get_course(item_location) + else: + item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only) + except ItemNotFoundError: + log.warning('Cannot find: %s', item_location) + return None + + return item + + def do_course_reindex(self, modulestore, course_key): + """ + (Re)index all content within the given course + """ + indexed_count = self.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True) + self._track_index_request('edx.course.index.reindexed', indexed_count) + return indexed_count + + +class LibrarySearchIndexer(SearchIndexerBase): + """ + Class to perform indexing for library search from different modulestores + """ + INDEX_NAME = "library_index" + DOCUMENT_TYPE = "library_content" + + def _get_structure_key(self, location): + """ Gets structure key from location """ + if isinstance(location, LibraryLocator): + course_key = location + else: + course_key = location.course_key.replace(version_guid=None, branch=None) + return course_key + + def _get_location_info(self, structure_key): + """ Builds location info dictionary """ + return {"library": unicode(structure_key)} + + def _id_modifier(self, usage_id): + """ Modifies usage_id to submit to index """ + return usage_id.replace(library_key=(usage_id.library_key.replace(version_guid=None, branch=None))) + + def _fetch_item(self, modulestore, item_location): + """ Fetch the item from the modulestore location, log if not found, but continue """ + try: + if isinstance(item_location, CourseLocator): + item = modulestore.get_library(item_location) + else: + item = modulestore.get_item(item_location) + except ItemNotFoundError: + log.warning('Cannot find: %s', item_location) + return None + + return item diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py index 7d84e5612451..7bb99b6ef79e 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py @@ -5,7 +5,7 @@ from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL from xmodule.exceptions import InvalidVersionError from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.courseware_index import CoursewareSearchIndexer +from xmodule.modulestore.search_index import get_indexer_for_location from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError from xmodule.modulestore.draft_and_published import ( ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError @@ -131,7 +131,12 @@ def update_item(self, descriptor, user_id, allow_not_found=False, force=False, * ) self._auto_publish_no_children(item.location, item.location.category, user_id, **kwargs) descriptor.location = old_descriptor_locn - return item + + indexer = get_indexer_for_location(item.location) + if indexer.index_on_update: + indexer.add_to_search_index(self, item.location) + + return item def create_item( self, user_id, course_key, block_type, block_id=None, @@ -152,7 +157,11 @@ def create_item( ) if not skip_auto_publish: self._auto_publish_no_children(item.location, item.location.category, user_id, **kwargs) - return item + + indexer = get_indexer_for_location(item.location) + if indexer.index_on_create: + indexer.add_to_search_index(self, item.location) + return item def create_child( self, user_id, parent_usage_key, block_type, block_id=None, @@ -209,9 +218,11 @@ def delete_item(self, location, user_id, revision=None, **kwargs): if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES: self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs) - # Remove this location from the courseware search index so that searches + # Remove this location from the search index so that searches # will refrain from showing it as a result - CoursewareSearchIndexer.add_to_search_index(self, location, delete=True) + indexer = get_indexer_for_location(location) + if indexer.index_on_delete: + indexer.add_to_search_index(self, location, delete=True) def _map_revision_to_branch(self, key, revision=None): """ @@ -357,7 +368,9 @@ def publish(self, location, user_id, blacklist=None, **kwargs): ) # Now it's been published, add the object to the courseware search index so that it appears in search results - CoursewareSearchIndexer.do_publish_index(self, location) + indexer = get_indexer_for_location(location) + if indexer.index_on_publish: + indexer.do_publish_index(self, location) return self.get_item(location.for_branch(ModuleStoreEnum.BranchName.published), **kwargs) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py b/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py new file mode 100644 index 000000000000..db16ff98295b --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_search_index.py @@ -0,0 +1,286 @@ +""" +Unit tests for search indexers +""" +from django.conf import settings +from django.test.utils import override_settings +from lazy.lazy import lazy +import os +import mock +import json + +from uuid import uuid4 +from datetime import datetime +from django.test import TestCase + +from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator, LibraryUsageLocator + +from xmodule.modulestore import ModuleStoreWriteBase +from xmodule.modulestore.search_index import get_indexer_for_location, CoursewareSearchIndexer, LibrarySearchIndexer + + +class TestIndexerForLocation(TestCase): + """ Tests for search indexers factory method """ + def test_given_course_locator_returns_courseware_indexer(self): + locator = mock.Mock(spec=CourseLocator) + indexer = get_indexer_for_location(locator) + self.assertIsInstance(indexer, CoursewareSearchIndexer) + + def test_given_library_locator_returns_library_indexer(self): + locator = mock.Mock(spec=LibraryLocator) + indexer = get_indexer_for_location(locator) + self.assertIsInstance(indexer, LibrarySearchIndexer) + + def test_given_course_item_locator_returns_courseware_indexer(self): + locator = mock.Mock(spec=BlockUsageLocator) + locator.course_key = mock.Mock(spec=CourseLocator) + indexer = get_indexer_for_location(locator) + self.assertIsInstance(indexer, CoursewareSearchIndexer) + + def test_given_library_item_locator_returns_library_indexer(self): + locator = mock.Mock(spec=LibraryUsageLocator) + locator.course_key = mock.Mock(spec=LibraryLocator) + indexer = get_indexer_for_location(locator) + self.assertIsInstance(indexer, LibrarySearchIndexer) + + +MOCK_SEARCH_ENGINE = getattr(settings, 'SEARCH_ENGINE', "search.tests.mock_search_engine.MockSearchEngine") + + +# pylint: disable=no-member +class SearchIndexerBaseTestMixin(object): + """ Common features for search indexers tests""" + ROOT_ID = None + modulestore = None + + TEST_INDEX_FILENAME = getattr( + settings, 'MOCK_SEARCH_BACKING_FILE', + "test_root/index_file.dat" + ) + + def setUp(self): + """ + Setting up mock search indexer backing file as prescribed by MOCK_SEARCH_BACKING_FILE setting + """ + super(SearchIndexerBaseTestMixin, self).setUp() + # this test implicitly uses MockSearchEngine, which in turn uses settings.MOCK_SEARCH_BACKING_FILE + # as backing file - so we create it + with open(self.TEST_INDEX_FILENAME, "w+") as index_file: + json.dump({}, index_file) + + self.modulestore = mock.Mock(spec=ModuleStoreWriteBase) + + def tearDown(self): + """ + Removing backing file created in set up + """ + super(SearchIndexerBaseTestMixin, self).tearDown() + os.remove(self.TEST_INDEX_FILENAME) + + def _make_location(self, block_type, course_id=ROOT_ID): # pylint: disable=unused-argument + """ Builds xblock location for specified block type and course id """ + raise NotImplementedError() + + def _process_index_id(self, location): # pylint: disable=unused-argument + """ Transforms block location to build correct index id """ + raise NotImplementedError() + + @lazy + def indexer(self): + """ Indexer under test instantiation """ + raise NotImplementedError() + + def _make_item(self, block_type, index_dictionary, course_id=None, start=None, children=None): + """ Builds XBlock mock with specified block_type, index disctionary, children, etc.""" + location = self._make_location(block_type, course_id) + result = mock.Mock() + if index_dictionary is not None: + result.index_dictionary = mock.Mock(return_value=index_dictionary) + else: + del result.index_dictionary + result.start = start + + result.children = children if children else [] + result.has_children = len(result.children) > 0 + result.location = location + result.scope_ids = mock.Mock() + result.scope_ids.usage_id = location + return result, location + + def _set_up_modulestore_get_item(self, items): + """ + Sets up mock modulestore get_item method to return only items listed in `items` parameter at specified locations + """ + def side_effect(loc, **kwargs): # pylint: disable=unused-argument + """ Side-effect method for mock - gets item by location """ + return items.get(loc, None) + self.modulestore.get_item = mock.Mock(side_effect=side_effect) + + def _make_index_entry(self, index_dict, location, start_date=None): + """ Helper method: builds index entry dictionary """ + result = { + 'id': self._process_index_id(location), + 'course': unicode(self.ROOT_ID), + } + result.update(index_dict) + if start_date: + result['start_date'] = start_date + return result + + @mock.patch(MOCK_SEARCH_ENGINE + '.remove') + @mock.patch(MOCK_SEARCH_ENGINE + '.index') + @mock.patch('xmodule.modulestore.search_index.SearchEngine.get_search_engine') + def test_no_searcher_does_nothing(self, search_engine, index, remove): + """ Tests that does indexer does nothing if no search engine is available """ + search_engine.return_value = None + _, location = self._make_item('html', {'item': 'value'}) + self.indexer.add_to_search_index(self.modulestore, location) + search_engine.assert_called_with(self.indexer.INDEX_NAME) + index.assert_not_called() + remove.assert_not_called() + + @mock.patch(MOCK_SEARCH_ENGINE + '.remove') + @mock.patch(MOCK_SEARCH_ENGINE + '.index') + def test_not_indexable_no_children_does_nothing(self, index, remove): + """ Tests that does indexer does nothing if xblock is not indexable """ + item, location = self._make_item('html', None) + self._set_up_modulestore_get_item({location: item}) + self.indexer.add_to_search_index(self.modulestore, location) + index.assert_not_called() + remove.assert_not_called() + + @mock.patch(MOCK_SEARCH_ENGINE + '.index') + def test_add_to_index_no_children_adds_to_index(self, patched_index): + """ Tests that indexer adds XBlock with no children to index """ + index_dict = {'item': 'value'} + item, location = self._make_item('html', index_dict) + self._set_up_modulestore_get_item({location: item}) + self.indexer.add_to_search_index(self.modulestore, location) + expected_index_entry = self._make_index_entry(index_dict, location) + patched_index.assert_called_with(self.indexer.DOCUMENT_TYPE, expected_index_entry) + + @mock.patch(MOCK_SEARCH_ENGINE + '.remove') + def test_add_to_index_with_delete_no_children_simple_removes_from_index(self, patched_remove): + """ Tests that indexer removes XBlock with no children from index """ + item, location = self._make_item('html', {'item': 'value'}) + self._set_up_modulestore_get_item({location: item}) + self.indexer.add_to_search_index(self.modulestore, location, delete=True) + patched_remove.assert_called_with(self.indexer.DOCUMENT_TYPE, self._process_index_id(location)) + + @mock.patch(MOCK_SEARCH_ENGINE + '.index') + def test_add_to_index_with_chidlren_adds_all_to_index(self, patched_index): + """ Tests that indexer adds XBlock with children to index (both Xblock and all of its children)""" + index_dict_child1, index_dict_child2 = {'child': 'child1'}, {'child': 'child2'} + child1, child_loc1 = self._make_item('text', index_dict_child1) + child2, child_loc2 = self._make_item('html', index_dict_child2) + + index_dict = {'item': 'value'} + start_date = datetime(2015, 7, 14, 22, 11, 03) + + item, location = self._make_item( + 'problem', index_dict, + children=[child_loc1, child_loc2], + start=start_date + ) + self._set_up_modulestore_get_item({location: item, child_loc1: child1, child_loc2: child2}) + self.indexer.add_to_search_index(self.modulestore, location) + + calls = patched_index.call_args_list + args, _ = calls[0] + self.assertEqual(args, ( + self.indexer.DOCUMENT_TYPE, + self._make_index_entry(index_dict_child1, child_loc1, start_date=start_date) + )) + args, _ = calls[1] + self.assertEqual(args, ( + self.indexer.DOCUMENT_TYPE, + self._make_index_entry(index_dict_child2, child_loc2, start_date=start_date) + )) + args, _ = calls[2] + self.assertEqual(args, ( + self.indexer.DOCUMENT_TYPE, + self._make_index_entry(index_dict, location, start_date=start_date) + )) + + @mock.patch(MOCK_SEARCH_ENGINE + '.remove') + def test_remove_from_index_with_chidlren_removes_all_from_index(self, patched_remove): + """ Tests that indexer removes XBlock with children from index (both Xblock and all of its children)""" + index_dict_child1, index_dict_child2 = {'child': 'child1'}, {'child': 'child2'} + child1, child_loc1 = self._make_item('text', index_dict_child1) + child2, child_loc2 = self._make_item('html', index_dict_child2) + + index_dict = {'item': 'value'} + + item, location = self._make_item('problem', index_dict, children=[child_loc1, child_loc2]) + self._set_up_modulestore_get_item({location: item, child_loc1: child1, child_loc2: child2}) + self.indexer.add_to_search_index(self.modulestore, location, delete=True) + + calls = patched_remove.call_args_list + args, _ = calls[0] + self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(child_loc1))) + args, _ = calls[1] + self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(child_loc2))) + args, _ = calls[2] + self.assertEqual(args, (self.indexer.DOCUMENT_TYPE, self._process_index_id(location))) + + +@override_settings(MOCK_SEARCH_ENGINE=MOCK_SEARCH_ENGINE) +class TestCoursewareSearchIndexer(SearchIndexerBaseTestMixin, TestCase): + """ Tests for CoursewareSearchIndexer """ + ROOT_ID = CourseLocator('testx', 'courseware_indexer_test', 'test_run') + + @lazy + def indexer(self): + """ Indexer under test instantiation """ + return CoursewareSearchIndexer() + + def _make_location(self, block_type, course_id=None): + """ Builds xblock location for specified block type and course id """ + course_id = course_id if course_id else self.ROOT_ID + return BlockUsageLocator(course_id, block_type, uuid4().hex) + + def _process_index_id(self, location): + """ Transforms block location to build correct index id """ + return unicode(location) + + def _make_index_entry(self, index_dict, location, start_date=None): + """ Helper method: builds index entry dictionary """ + result = { + 'id': self._process_index_id(location), + 'course': unicode(self.ROOT_ID), + } + result.update(index_dict) + if start_date: + result['start_date'] = start_date + return result + + +@override_settings(MOCK_SEARCH_ENGINE=MOCK_SEARCH_ENGINE) +class TestLibrarySearchIndexer(SearchIndexerBaseTestMixin, TestCase): + """ Tests for LibrarySearchIndexer """ + ROOT_ID = LibraryLocator('lib', 'Lib1') + + @lazy + def indexer(self): + """ Indexer under test instantiation """ + return LibrarySearchIndexer() + + def _make_location(self, block_type, course_id=None): + """ Builds xblock location for specified block type and course id """ + course_id = course_id if course_id else self.ROOT_ID + return LibraryUsageLocator(course_id, block_type, uuid4().hex) + + def _process_index_id(self, location): + """ Transforms block location to build correct index id """ + new_loc = location.replace(library_key=location.library_key.replace(version_guid=None, branch=None)) + return unicode(new_loc) + + def _make_index_entry(self, index_dict, location, start_date=None): + """ Helper method: builds index entry dictionary """ + result = { + 'id': self._process_index_id(location), + 'library': unicode(self.ROOT_ID), + } + result.update(index_dict) + if start_date: + result['start_date'] = start_date + return result diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py index d1309fded457..3a58a118d6f4 100644 --- a/common/lib/xmodule/xmodule/tests/test_capa_module.py +++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py @@ -1666,18 +1666,26 @@ def test_check_unmask_answerpool(self): @ddt.ddt class CapaDescriptorTest(unittest.TestCase): - def _create_descriptor(self, xml): + def _create_descriptor(self, xml, name=None): """ Creates a CapaDescriptor to run test against """ descriptor = CapaDescriptor(get_test_system(), scope_ids=1) descriptor.data = xml + if name: + descriptor.display_name = name return descriptor @ddt.data(*responsetypes.registry.registered_tags()) def test_all_response_types(self, response_tag): """ Tests that every registered response tag is correctly returned """ xml = "<{response_tag}>".format(response_tag=response_tag) - descriptor = self._create_descriptor(xml) + name = "Some Capa Problem" + descriptor = self._create_descriptor(xml, name=name) self.assertEquals(descriptor.problem_types, {response_tag}) + self.assertEquals(descriptor.index_dictionary(), { + 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE, + 'display_name': name, + 'problem_types': [response_tag] + }) def test_response_types_ignores_non_response_tags(self): xml = textwrap.dedent(""" @@ -1694,8 +1702,14 @@ def test_response_types_ignores_non_response_tags(self): """) - descriptor = self._create_descriptor(xml) + name = "Test Capa Problem" + descriptor = self._create_descriptor(xml, name=name) self.assertEquals(descriptor.problem_types, {"multiplechoiceresponse"}) + self.assertEquals(descriptor.index_dictionary(), { + 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE, + 'display_name': name, + 'problem_types': ["multiplechoiceresponse"] + }) def test_response_types_multiple_tags(self): xml = textwrap.dedent(""" @@ -1717,8 +1731,16 @@ def test_response_types_multiple_tags(self): """) - descriptor = self._create_descriptor(xml) + name = "Other Test Capa Problem" + descriptor = self._create_descriptor(xml, name=name) self.assertEquals(descriptor.problem_types, {"multiplechoiceresponse", "optionresponse"}) + self.assertEquals( + descriptor.index_dictionary(), { + 'content_type': CapaDescriptor.INDEX_CONTENT_TYPE, + 'display_name': name, + 'problem_types': ["optionresponse", "multiplechoiceresponse"] + } + ) class ComplexEncoderTest(unittest.TestCase): diff --git a/common/lib/xmodule/xmodule/tests/test_library_content.py b/common/lib/xmodule/xmodule/tests/test_library_content.py index fd9313f8463c..cbe9ad301867 100644 --- a/common/lib/xmodule/xmodule/tests/test_library_content.py +++ b/common/lib/xmodule/xmodule/tests/test_library_content.py @@ -18,6 +18,7 @@ from xmodule.tests import get_test_system from xmodule.validation import StudioValidationMessage from xmodule.x_module import AUTHOR_VIEW +from search.search_engine_base import SearchEngine dummy_render = lambda block, _: Fragment(block.data) # pylint: disable=invalid-name @@ -66,10 +67,17 @@ def get_module(descriptor): module.xmodule_runtime = module_system -class TestLibraryContentModule(LibraryContentTest): +class LibraryContentModuleTestMixin(object): """ Basic unit tests for LibraryContentModule """ + problem_types = [ + ["multiplechoiceresponse"], ["optionresponse"], ["optionresponse", "coderesponse"], + ["coderesponse", "optionresponse"] + ] + + problem_type_lookup = {} + def _get_capa_problem_type_xml(self, *args): """ Helper function to create empty CAPA problem definition """ problem = "" @@ -84,12 +92,10 @@ def _create_capa_problems(self): Creates four blocks total. """ - problem_types = [ - ["multiplechoiceresponse"], ["optionresponse"], ["optionresponse", "coderesponse"], - ["coderesponse", "optionresponse"] - ] - for problem_type in problem_types: - self.make_block("problem", self.library, data=self._get_capa_problem_type_xml(*problem_type)) + self.problem_type_lookup = {} + for problem_type in self.problem_types: + block = self.make_block("problem", self.library, data=self._get_capa_problem_type_xml(*problem_type)) + self.problem_type_lookup[block.location] = problem_type def test_lib_content_block(self): """ @@ -236,6 +242,42 @@ def test_non_editable_settings(self): self.assertNotIn(LibraryContentDescriptor.display_name, non_editable_metadata_fields) +@patch('xmodule.library_tools.SearchEngine.get_search_engine', Mock(return_value=None)) +class TestLibraryContentModuleNoSearchIndex(LibraryContentModuleTestMixin, LibraryContentTest): + """ + Tests for library container when no search index is available. + Tests fallback low-level CAPA problem introspection + """ + pass + + +search_index_mock = Mock(spec=SearchEngine) # pylint: disable=invalid-name + + +@patch('xmodule.library_tools.SearchEngine.get_search_engine', Mock(return_value=search_index_mock)) +class TestLibraryContentModuleWithSearchIndex(LibraryContentModuleTestMixin, LibraryContentTest): + """ + Tests for library container with mocked search engine response. + """ + def _get_search_response(self, field_dictionary=None): + """ Mocks search response as returned by search engine """ + target_type = field_dictionary.get('problem_types') + matched_block_locations = [ + key for key, problem_types in + self.problem_type_lookup.items() if target_type in problem_types + ] + return { + 'results': [ + {'data': {'id': str(location)}} for location in matched_block_locations + ] + } + + def setUp(self): + """ Sets up search engine mock """ + super(TestLibraryContentModuleWithSearchIndex, self).setUp() + search_index_mock.search = Mock(side_effect=self._get_search_response) + + @patch( 'xmodule.modulestore.split_mongo.caching_descriptor_system.CachingDescriptorSystem.render', VanillaRuntime.render ) diff --git a/common/test/acceptance/tests/helpers.py b/common/test/acceptance/tests/helpers.py index 48204fb575cf..fcf3eefe8fab 100644 --- a/common/test/acceptance/tests/helpers.py +++ b/common/test/acceptance/tests/helpers.py @@ -417,3 +417,17 @@ def create_user_partition_json(partition_id, name, description, groups, scheme=" return UserPartition( partition_id, name, description, groups, MockUserPartitionScheme(scheme) ).to_json() + + +class TestWithSearchIndexMixin(object): + """ Mixin encapsulating search index creation """ + TEST_INDEX_FILENAME = "test_root/index_file.dat" + + def _create_search_index(self): + """ Creates search index backing file """ + with open(self.TEST_INDEX_FILENAME, "w+") as index_file: + json.dump({}, index_file) + + def _cleanup_index_file(self): + """ Removes search index backing file """ + os.remove(self.TEST_INDEX_FILENAME) diff --git a/common/test/acceptance/tests/lms/test_library.py b/common/test/acceptance/tests/lms/test_library.py index e4bb1cd8ec09..143acc77f685 100644 --- a/common/test/acceptance/tests/lms/test_library.py +++ b/common/test/acceptance/tests/lms/test_library.py @@ -6,7 +6,7 @@ import textwrap from nose.plugins.attrib import attr -from ..helpers import UniqueCourseTest +from ..helpers import UniqueCourseTest, TestWithSearchIndexMixin from ...pages.studio.auto_auth import AutoAuthPage from ...pages.studio.overview import CourseOutlinePage from ...pages.studio.library import StudioLibraryContentEditor, StudioLibraryContainerXBlockWrapper @@ -196,10 +196,19 @@ def test_shows_all_if_max_set_to_greater_value(self): @ddt.ddt @attr('shard_3') -class StudioLibraryContainerCapaFilterTest(LibraryContentTestBase): +class StudioLibraryContainerCapaFilterTest(LibraryContentTestBase, TestWithSearchIndexMixin): """ Test Library Content block in LMS """ + def setUp(self): + """ SetUp method """ + self._create_search_index() + super(StudioLibraryContainerCapaFilterTest, self).setUp() + + def tearDown(self): + self._cleanup_index_file() + super(StudioLibraryContainerCapaFilterTest, self).tearDown() + def _get_problem_choice_group_text(self, name, items): """ Generates Choice Group CAPA problem XML """ items_text = "\n".join([ @@ -231,7 +240,7 @@ def populate_library_fixture(self, library_fixture): """ Populates library fixture with XBlock Fixtures """ - library_fixture.add_children( + items = ( XBlockFixtureDesc( "problem", "Problem Choice Group 1", data=self._get_problem_choice_group_text("Problem Choice Group 1 Text", [("1", False), ('2', True)]) @@ -249,6 +258,7 @@ def populate_library_fixture(self, library_fixture): data=self._get_problem_select_text("Problem Select 2 Text", ["Option 3", "Option 4"], "Option 4") ), ) + library_fixture.add_children(*items) @property def _problem_headers(self): diff --git a/common/test/acceptance/tests/studio/test_studio_library_container.py b/common/test/acceptance/tests/studio/test_studio_library_container.py index 772b9013a001..20a4f111b67f 100644 --- a/common/test/acceptance/tests/studio/test_studio_library_container.py +++ b/common/test/acceptance/tests/studio/test_studio_library_container.py @@ -7,7 +7,7 @@ from .base_studio_test import StudioLibraryTest from ...fixtures.course import CourseFixture -from ..helpers import UniqueCourseTest +from ..helpers import UniqueCourseTest, TestWithSearchIndexMixin from ...pages.studio.library import StudioLibraryContentEditor, StudioLibraryContainerXBlockWrapper from ...pages.studio.overview import CourseOutlinePage from ...fixtures.course import XBlockFixtureDesc @@ -18,7 +18,7 @@ @ddt.ddt -class StudioLibraryContainerTest(StudioLibraryTest, UniqueCourseTest): +class StudioLibraryContainerTest(StudioLibraryTest, UniqueCourseTest, TestWithSearchIndexMixin): """ Test Library Content block in LMS """ @@ -26,6 +26,7 @@ def setUp(self): """ Install library with some content and a course using fixtures """ + self._create_search_index() super(StudioLibraryContainerTest, self).setUp() # Also create a course: self.course_fixture = CourseFixture( @@ -42,6 +43,11 @@ def setUp(self): subsection = self.outline.section(SECTION_NAME).subsection(SUBSECTION_NAME) self.unit_page = subsection.expand_subsection().unit(UNIT_NAME).go_to() + def tearDown(self): + """ Cleans up search index backing file """ + self._cleanup_index_file() + super(StudioLibraryContainerTest, self).tearDown() + def populate_library_fixture(self, library_fixture): """ Populate the children of the test course fixture. diff --git a/lms/envs/test.py b/lms/envs/test.py index 420449698893..6af25730528a 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -462,6 +462,9 @@ FEATURES['ENABLE_COURSEWARE_SEARCH'] = True # Use MockSearchEngine as the search engine for test scenario SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine" +MOCK_SEARCH_BACKING_FILE = ( + TEST_ROOT / "index_file.dat" # pylint: disable=no-value-for-parameter +).abspath() FACEBOOK_APP_SECRET = "Test" FACEBOOK_APP_ID = "Test" diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 2c3c0ac1766f..e7c8f9d46ebf 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -38,7 +38,7 @@ git+https://github.com/mitocw/django-cas.git@60a5b8e5a62e63e0d5d224a87f0b489201a -e git+https://github.com/edx/edx-val.git@64aa7637e3459fb3000a85a9e156880a40307dd1#egg=edx-val -e git+https://github.com/pmitros/RecommenderXBlock.git@9b07e807c89ba5761827d0387177f71aa57ef056#egg=recommender-xblock -e git+https://github.com/edx/edx-milestones.git@547f2250ee49e73ce8d7ff4e78ecf1b049892510#egg=edx-milestones --e git+https://github.com/edx/edx-search.git@21ac6b06b3bfe789dcaeaf4e2ab5b00a688324d4#egg=edx-search +-e git+https://github.com/edx/edx-search.git@b5444b2702817bc04f5d0d025779ee50dbdec465#egg=edx-search git+https://github.com/edx/edx-lint.git@8bf82a32ecb8598c415413df66f5232ab8d974e9#egg=edx_lint==0.2.1 -e git+https://github.com/edx/xblock-utils.git@17e247d66fabb53f0453515b093a030ee345a1b0#egg=xblock-utils -e git+https://github.com/edx-solutions/xblock-google-drive.git@138e6fa0bf3a2013e904a085b9fed77dab7f3f21#egg=xblock-google-drive