Eugeny/problem type search

cms/djangoapps/contentstore/courseware_index.py

@@ -1,20 +1,17 @@
 """ Code to allow module store to interface with courseware index """
 from __future__ import absolute_import
-
+from abc import ABCMeta, abstractmethod
 from datetime import timedelta
 import logging
+from six import add_metaclass
 
 from django.conf import settings
 from django.utils.translation import ugettext as _
 from eventtracking import tracker
 from xmodule.modulestore import ModuleStoreEnum
+from xmodule.library_tools import normalize_key_for_search
 from search.search_engine_base import SearchEngine
 
-
-# Use default index and document names for now
-INDEX_NAME = "courseware_index"
-DOCUMENT_TYPE = "courseware_content"
-
 # REINDEX_AGE is the default amount of time that we look back for changes
 # that might have happened. If we are provided with a time at which the
 # indexing is triggered, then we know it is safe to only index items
@@ -25,13 +22,6 @@
 log = logging.getLogger('edx.modulestore')
 
 
-def indexing_is_enabled():
-    """
-    Checks to see if the indexing feature is enabled
-    """
-    return settings.FEATURES.get('ENABLE_COURSEWARE_INDEX', False)
-
-
 class SearchIndexingError(Exception):
     """ Indicates some error(s) occured during indexing """
 
@@ -40,18 +30,71 @@ def __init__(self, message, error_list):
         self.error_list = error_list
 
 
-class CoursewareSearchIndexer(object):
+@add_metaclass(ABCMeta)
+class SearchIndexerBase(object):
     """
-    Class to perform indexing for courseware search from different modulestores
+    Base class to perform indexing for courseware or library search from different modulestores
     """
+    __metaclass__ = ABCMeta
+
+    INDEX_NAME = None
+    DOCUMENT_TYPE = None
+    ENABLE_INDEXING_KEY = None
+
+    INDEX_EVENT = {
+        'name': None,
+        'category': None
+    }
+
+    @classmethod
+    def indexing_is_enabled(cls):
+        """
+        Checks to see if the indexing feature is enabled
+        """
+        return settings.FEATURES.get(cls.ENABLE_INDEXING_KEY, False)
+
+    @classmethod
+    @abstractmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+
+    @classmethod
+    @abstractmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+
+    @classmethod
+    @abstractmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+
+    @classmethod
+    def _id_modifier(cls, usage_id):
+        """ Modifies usage_id to submit to index """
+        return usage_id
 
     @classmethod
-    def index_course(cls, modulestore, course_key, triggered_at=None, reindex_age=REINDEX_AGE):
+    def remove_deleted_items(cls, searcher, structure_key, exclude_items):
+        """
+        remove any item that is present in the search index that is not present in updated list of indexed items
+        as we find items we can shorten the set of items to keep
+        """
+        response = searcher.search(
+            doc_type=cls.DOCUMENT_TYPE,
+            field_dictionary=cls._get_location_info(structure_key),
+            exclude_ids=exclude_items
+        )
+        result_ids = [result["data"]["id"] for result in response["results"]]
+        for result_id in result_ids:
+            searcher.remove(cls.DOCUMENT_TYPE, result_id)
+
+    @classmethod
+    def index(cls, modulestore, structure_key, triggered_at=None, reindex_age=REINDEX_AGE):
         """
         Process course for indexing
 
         Arguments:
-        course_key (CourseKey) - course identifier
+        structure_key (CourseKey|LibraryKey) - course or library identifier
 
         triggered_at (datetime) - provides time at which indexing was triggered;
             useful for index updates - only things changed recently from that date
@@ -64,13 +107,12 @@ def index_course(cls, modulestore, course_key, triggered_at=None, reindex_age=RE
         Number of items that have been added to the index
         """
         error_list = []
-        searcher = SearchEngine.get_search_engine(INDEX_NAME)
+        searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
         if not searcher:
             return
 
-        location_info = {
-            "course": unicode(course_key),
-        }
+        structure_key = cls.normalize_structure_key(structure_key)
+        location_info = cls._get_location_info(structure_key)
 
         # Wrap counter in dictionary - otherwise we seem to lose scope inside the embedded function `index_item`
         indexed_count = {
@@ -101,7 +143,7 @@ def index_item(item, skip_index=False):
             if not item_index_dictionary and not item.has_children:
                 return
 
-            item_id = unicode(item.scope_ids.usage_id)
+            item_id = unicode(cls._id_modifier(item.scope_ids.usage_id))
             indexed_items.add(item_id)
             if item.has_children:
                 # determine if it's okay to skip adding the children herein based upon how recently any may have changed
@@ -122,38 +164,24 @@ def index_item(item, skip_index=False):
                 if item.start:
                     item_index['start_date'] = item.start
 
-                searcher.index(DOCUMENT_TYPE, item_index)
+                searcher.index(cls.DOCUMENT_TYPE, item_index)
                 indexed_count["count"] += 1
             except Exception as err:  # pylint: disable=broad-except
                 # broad exception so that index operation does not fail on one item of many
                 log.warning('Could not index item: %s - %r', item.location, err)
                 error_list.append(_('Could not index item: {}').format(item.location))
 
-        def remove_deleted_items():
-            """
-            remove any item that is present in the search index that is not present in updated list of indexed items
-            as we find items we can shorten the set of items to keep
-            """
-            response = searcher.search(
-                doc_type=DOCUMENT_TYPE,
-                field_dictionary={"course": unicode(course_key)},
-                exclude_ids=indexed_items
-            )
-            result_ids = [result["data"]["id"] for result in response["results"]]
-            for result_id in result_ids:
-                searcher.remove(DOCUMENT_TYPE, result_id)
-
         try:
             with modulestore.branch_setting(ModuleStoreEnum.RevisionOption.published_only):
-                course = modulestore.get_course(course_key, depth=None)
-                for item in course.get_children():
+                structure = cls._fetch_top_level(modulestore, structure_key)
+                for item in structure.get_children():
                     index_item(item)
-                remove_deleted_items()
+                cls.remove_deleted_items(searcher, structure_key, indexed_items)
         except Exception as err:  # pylint: disable=broad-except
             # broad exception so that index operation does not prevent the rest of the application from working
             log.exception(
                 "Indexing error encountered, courseware index may be out of date %s - %r",
-                course_key,
+                structure_key,
                 err
             )
             error_list.append(_('General indexing error occurred'))
@@ -164,31 +192,111 @@ def remove_deleted_items():
         return indexed_count["count"]
 
     @classmethod
-    def do_course_reindex(cls, modulestore, course_key):
+    def _do_reindex(cls, modulestore, structure_key):
         """
-        (Re)index all content within the given course, tracking the fact that a full reindex has taking place
+        (Re)index all content within the given structure (course or library),
+        tracking the fact that a full reindex has taken place
         """
-        indexed_count = cls.index_course(modulestore, course_key)
+        indexed_count = cls.index(modulestore, structure_key)
         if indexed_count:
-            cls._track_index_request('edx.course.index.reindexed', indexed_count)
+            cls._track_index_request(cls.INDEX_EVENT['name'], cls.INDEX_EVENT['category'], indexed_count)
         return indexed_count
 
     @classmethod
-    def _track_index_request(cls, event_name, indexed_count):
+    def _track_index_request(cls, event_name, category, indexed_count):
         """Track content index requests.
 
         Arguments:
             event_name (str):  Name of the event to be logged.
+            category (str): category of indexed items
+            indexed_count (int): number of indexed items
         Returns:
             None
 
         """
         data = {
             "indexed_count": indexed_count,
-            'category': 'courseware_index',
+            'category': category,
         }
 
         tracker.emit(
             event_name,
             data
         )
+
+
+class CoursewareSearchIndexer(SearchIndexerBase):
+    """
+    Class to perform indexing for courseware search from different modulestores
+    """
+    INDEX_NAME = "courseware_index"
+    DOCUMENT_TYPE = "courseware_content"
+    ENABLE_INDEXING_KEY = 'ENABLE_COURSEWARE_INDEX'
+
+    INDEX_EVENT = {
+        'name': 'edx.course.index.reindexed',
+        'category': 'courseware_index'
+    }
+
+    @classmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+        return structure_key
+
+    @classmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+        return modulestore.get_course(structure_key, depth=None)
+
+    @classmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+        return {"course": unicode(normalized_structure_key)}
+
+    @classmethod
+    def do_course_reindex(cls, modulestore, course_key):
+        """
+        (Re)index all content within the given course, tracking the fact that a full reindex has taken place
+        """
+        return cls._do_reindex(modulestore, course_key)
+
+
+class LibrarySearchIndexer(SearchIndexerBase):
+    """
+    Base class to perform indexing for library search from different modulestores
+    """
+    INDEX_NAME = "library_index"
+    DOCUMENT_TYPE = "library_content"
+    ENABLE_INDEXING_KEY = 'ENABLE_LIBRARY_INDEX'
+
+    INDEX_EVENT = {
+        'name': 'edx.library.index.reindexed',
+        'category': 'library_index'
+    }
+
+    @classmethod
+    def normalize_structure_key(cls, structure_key):
+        """ Normalizes structure key for use in indexing """
+        return normalize_key_for_search(structure_key)
+
+    @classmethod
+    def _fetch_top_level(cls, modulestore, structure_key):
+        """ Fetch the item from the modulestore location """
+        return modulestore.get_library(structure_key, depth=None)
+
+    @classmethod
+    def _get_location_info(cls, normalized_structure_key):
+        """ Builds location info dictionary """
+        return {"library": unicode(normalized_structure_key)}
+
+    @classmethod
+    def _id_modifier(cls, usage_id):
+        """ Modifies usage_id to submit to index """
+        return usage_id.replace(library_key=(usage_id.library_key.replace(version_guid=None, branch=None)))
+
+    @classmethod
+    def do_library_reindex(cls, modulestore, library_key):
+        """
+        (Re)index all content within the given library, tracking the fact that a full reindex has taken place
+        """
+        return cls._do_reindex(modulestore, library_key)

cms/djangoapps/contentstore/management/commands/reindex_library.py

@@ -0,0 +1,75 @@
+""" Management command to update libraries' search index """
+from django.core.management import BaseCommand, CommandError
+from optparse import make_option
+from textwrap import dedent
+
+from contentstore.courseware_index import LibrarySearchIndexer
+
+from opaque_keys.edx.keys import CourseKey
+from opaque_keys import InvalidKeyError
+from opaque_keys.edx.locations import SlashSeparatedCourseKey
+from opaque_keys.edx.locator import LibraryLocator
+
+from .prompt import query_yes_no
+
+from xmodule.modulestore.django import modulestore
+
+
+class Command(BaseCommand):
+    """
+    Command to reindex content libraries (single, multiple or all available)
+
+    Examples:
+
+        ./manage.py reindex_library lib1 lib2 - reindexes libraries with keys lib1 and lib2
+        ./manage.py reindex_library --all - reindexes all available libraries
+    """
+    help = dedent(__doc__)
+
+    can_import_settings = True
+
+    args = "<library_id library_id ...>"
+
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '--all',
+            action='store_true',
+            dest='all',
+            default=False,
+            help='Reindex all libraries'
+        ),)
+
+    CONFIRMATION_PROMPT = u"Reindexing all libraries might be a time consuming operation. Do you want to continue?"
+
+    def _parse_library_key(self, raw_value):
+        """ Parses library key from string """
+        try:
+            result = CourseKey.from_string(raw_value)
+        except InvalidKeyError:
+            result = SlashSeparatedCourseKey.from_deprecated_string(raw_value)
+
+        if not isinstance(result, LibraryLocator):
+            raise CommandError(u"Argument {0} is not a library key".format(raw_value))
+
+        return result
+
+    def handle(self, *args, **options):
+        """
+        By convention set by django developers, this method actually executes command's actions.
+        So, there could be no better docstring than emphasize this once again.
+        """
+        if len(args) == 0 and not options.get('all', False):
+            raise CommandError(u"reindex_library requires one or more arguments: <library_id>")
+
+        store = modulestore()
+
+        if options.get('all', False):
+            if query_yes_no(self.CONFIRMATION_PROMPT, default="no"):
+                library_keys = [library.location.library_key.replace(branch=None) for library in store.get_libraries()]
+            else:
+                return
+        else:
+            library_keys = map(self._parse_library_key, args)
+
+        for library_key in library_keys:
+            LibrarySearchIndexer.do_library_reindex(store, library_key)