Content libraries analytics enhancements (SOL-121)

cms/djangoapps/contentstore/features/courses.py

@@ -34,9 +34,8 @@ def i_create_a_course(step):
     create_a_course()
 
 
-# pylint: disable=invalid-name
 @step('I click the course link in Studio Home$')
-def i_click_the_course_link_in_studio_home(step):
+def i_click_the_course_link_in_studio_home(step):  # pylint: disable=invalid-name
     course_css = 'a.course-link'
     world.css_click(course_css)
 

common/djangoapps/terrain/ui_helpers.py

@@ -28,28 +28,27 @@
 
 REQUIREJS_WAIT = {
     # Settings - Schedule & Details
-    re.compile('^Schedule & Details Settings \|'): [
+    re.compile(r'^Schedule & Details Settings \|'): [
         "jquery", "js/base", "js/models/course",
         "js/models/settings/course_details", "js/views/settings/main"],
 
     # Settings - Advanced Settings
-    re.compile('^Advanced Settings \|'): [
+    re.compile(r'^Advanced Settings \|'): [
         "jquery", "js/base", "js/models/course", "js/models/settings/advanced",
         "js/views/settings/advanced", "codemirror"],
 
     # Unit page
-    re.compile('^Unit \|'): [
+    re.compile(r'^Unit \|'): [
         "jquery", "js/base", "js/models/xblock_info", "js/views/pages/container",
         "js/collections/component_template", "xmodule", "coffee/src/main", "xblock/cms.runtime.v1"],
 
     # Content - Outline
     # Note that calling your org, course number, or display name, 'course' will mess this up
-    re.compile('^Course Outline \|'): [
+    re.compile(r'^Course Outline \|'): [
         "js/base", "js/models/course", "js/models/location", "js/models/section"],
 
     # Dashboard
-    # pylint: disable=anomalous-backslash-in-string
-    re.compile('^Studio Home \|'): [
+    re.compile(r'^Studio Home \|'): [
         "js/sock", "gettext", "js/base",
         "jquery.ui", "coffee/src/main", "underscore"],
 
@@ -60,7 +59,7 @@
     ],
 
     # Pages
-    re.compile('^Pages \|'): [
+    re.compile(r'^Pages \|'): [
         'js/models/explicit_url', 'coffee/src/views/tabs',
         'xmodule', 'coffee/src/main', 'xblock/cms.runtime.v1'
     ],

common/lib/xmodule/xmodule/library_content_module.py

@@ -220,25 +220,56 @@ def selected_children(self):
         if hasattr(self, "_selected_set"):
             # Already done:
             return self._selected_set  # pylint: disable=access-member-before-definition
+
+        selected = set(tuple(k) for k in self.selected)  # set of (block_type, block_id) tuples assigned to this student
+        previous_count = len(selected)
+
+        lib_tools = self.runtime.service(self, 'library_tools')
+        format_block_keys = lambda keys: lib_tools.create_block_analytics_summary(self.location.course_key, keys)
+
+        def publish_event(event_name, **kwargs):
+            """ Publish an event for analytics purposes """
+            event_data = {
+                "location": unicode(self.location),
+                "result": format_block_keys(selected),
+                "previous_count": previous_count,
+                "max_count": self.max_count,
+            }
+            event_data.update(kwargs)
+            self.runtime.publish(self, "edx.librarycontentblock.content.{}".format(event_name), event_data)
+
         # Determine which of our children we will show:
-        selected = set(tuple(k) for k in self.selected)  # set of (block_type, block_id) tuples
         valid_block_keys = set([(c.block_type, c.block_id) for c in self.children])  # pylint: disable=no-member
         # Remove any selected blocks that are no longer valid:
-        selected -= (selected - valid_block_keys)
+        invalid_block_keys = (selected - valid_block_keys)
+        if invalid_block_keys:
+            selected -= invalid_block_keys
+            # Publish an event for analytics purposes:
+            # reason "invalid" means deleted from library or a different library is now being used.
+            publish_event("removed", removed=format_block_keys(invalid_block_keys), reason="invalid")
         # If max_count has been decreased, we may have to drop some previously selected blocks:
+        overlimit_block_keys = set()
         while len(selected) > self.max_count:
-            selected.pop()
+            overlimit_block_keys.add(selected.pop())
+        if overlimit_block_keys:
+            # Publish an event for analytics purposes:
+            publish_event("removed", removed=format_block_keys(overlimit_block_keys), reason="overlimit")
         # Do we have enough blocks now?
         num_to_add = self.max_count - len(selected)
         if num_to_add > 0:
+            added_block_keys = None
             # We need to select [more] blocks to display to this user:
+            pool = valid_block_keys - selected
             if self.mode == "random":
-                pool = valid_block_keys - selected
                 num_to_add = min(len(pool), num_to_add)
-                selected |= set(random.sample(pool, num_to_add))
+                added_block_keys = set(random.sample(pool, num_to_add))
                 # We now have the correct n random children to show for this user.
             else:
                 raise NotImplementedError("Unsupported mode.")
+            selected |= added_block_keys
+            if added_block_keys:
+                # Publish an event for analytics purposes:
+                publish_event("assigned", added=format_block_keys(added_block_keys))
         # Save our selections to the user state, to ensure consistency:
         self.selected = list(selected)  # TODO: this doesn't save from the LMS "Progress" page.
         # Cache the results

common/lib/xmodule/xmodule/library_tools.py

@@ -44,6 +44,44 @@ def get_library_version(self, lib_key):
             return library.location.library_key.version_guid
         return None
 
+    def create_block_analytics_summary(self, course_key, block_keys):
+        """
+        Given a CourseKey and a list of (block_type, block_id) pairs,
+        prepare the JSON-ready metadata needed for analytics logging.
+
+        This is [
+            {"usage_key": x, "original_usage_key": y, "original_usage_version": z, "descendants": [...]}
+        ]
+        where the main list contains all top-level blocks, and descendants contains a *flat* list of all
+        descendants of the top level blocks, if any.
+        """
+        def summarize_block(usage_key):
+            """ Basic information about the given block """
+            orig_key, orig_version = self.store.get_block_original_usage(usage_key)
+            return {
+                "usage_key": unicode(usage_key),
+                "original_usage_key": unicode(orig_key) if orig_key else None,
+                "original_usage_version": unicode(orig_version) if orig_version else None,
+            }
+
+        result_json = []
+        for block_key in block_keys:
+            key = course_key.make_usage_key(*block_key)
+            info = summarize_block(key)
+            info['descendants'] = []
+            try:
+                block = self.store.get_item(key, depth=None)  # Load the item and all descendants
+                children = list(getattr(block, "children", []))
+                while children:
+                    child_key = children.pop()
+                    child = self.store.get_item(child_key)
+                    info['descendants'].append(summarize_block(child_key))
+                    children.extend(getattr(child, "children", []))
+            except ItemNotFoundError:
+                pass  # The block has been deleted
+            result_json.append(info)
+        return result_json
+
     def _filter_child(self, usage_key, capa_type):
         """
         Filters children by CAPA problem type, if configured

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -509,6 +509,18 @@ def get_parent_location(self, location, **kwargs):
         store = self._get_modulestore_for_courseid(location.course_key)
         return store.get_parent_location(location, **kwargs)
 
+    def get_block_original_usage(self, usage_key):
+        """
+        If a block was inherited into another structure using copy_from_template,
+        this will return the original block usage locator from which the
+        copy was inherited.
+        """
+        try:
+            store = self._verify_modulestore_support(usage_key.course_key, 'get_block_original_usage')
+            return store.get_block_original_usage(usage_key)
+        except NotImplementedError:
+            return None, None
+
     def get_modulestore_type(self, course_id):
         """
         Returns a type which identifies which modulestore is servicing the given course_id.

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -454,12 +454,17 @@ def version_block(self, block_info, user_id, update_version):
         if block_info['edit_info'].get('update_version') == update_version:
             return
 
+        original_usage = block_info['edit_info'].get('original_usage')
+        original_usage_version = block_info['edit_info'].get('original_usage_version')
         block_info['edit_info'] = {
             'edited_on': datetime.datetime.now(UTC),
             'edited_by': user_id,
             'previous_version': block_info['edit_info']['update_version'],
             'update_version': update_version,
         }
+        if original_usage:
+            block_info['edit_info']['original_usage'] = original_usage
+            block_info['edit_info']['original_usage_version'] = original_usage_version
 
     def find_matching_course_indexes(self, branch=None, search_targets=None):
         """
@@ -1254,6 +1259,21 @@ def get_definition_successors(self, definition_locator, version_history_depth=1)
         # TODO implement
         pass
 
+    def get_block_original_usage(self, usage_key):
+        """
+        If a block was inherited into another structure using copy_from_template,
+        this will return the original block usage locator and version from
+        which the copy was inherited.
+
+        Returns usage_key, version if the data is available, otherwise returns (None, None)
+        """
+        blocks = self._lookup_course(usage_key.course_key).structure['blocks']
+        block = blocks.get(BlockKey.from_usage_key(usage_key))
+        if block and 'original_usage' in block['edit_info']:
+            usage_key = BlockUsageLocator.from_string(block['edit_info']['original_usage'])
+            return usage_key, block['edit_info'].get('original_usage_version')
+        return None, None
+
     def create_definition_from_data(self, course_key, new_def_data, category, user_id):
         """
         Pull the definition fields out of descriptor and save to the db as a new definition
@@ -2214,6 +2234,8 @@ def _copy_from_template(self, source_structures, source_keys, dest_structure, ne
             # Setting it to the source_block_info structure version here breaks split_draft's has_changes() method.
             new_block_info['edit_info']['edited_by'] = user_id
             new_block_info['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+            new_block_info['edit_info']['original_usage'] = unicode(usage_key.replace(branch=None, version_guid=None))
+            new_block_info['edit_info']['original_usage_version'] = source_block_info['edit_info'].get('update_version')
             dest_structure['blocks'][new_block_key] = new_block_info
 
             children = source_block_info['fields'].get('children')

common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py

@@ -268,6 +268,15 @@ def get_parent_location(self, location, revision=None, **kwargs):
         location = self._map_revision_to_branch(location, revision=revision)
         return super(DraftVersioningModuleStore, self).get_parent_location(location, **kwargs)
 
+    def get_block_original_usage(self, usage_key):
+        """
+        If a block was inherited into another structure using copy_from_template,
+        this will return the original block usage locator from which the
+        copy was inherited.
+        """
+        usage_key = self._map_revision_to_branch(usage_key)
+        return super(DraftVersioningModuleStore, self).get_block_original_usage(usage_key)
+
     def get_orphans(self, course_key, **kwargs):
         course_key = self._map_revision_to_branch(course_key)
         return super(DraftVersioningModuleStore, self).get_orphans(course_key, **kwargs)

common/lib/xmodule/xmodule/modulestore/tests/utils.py

@@ -119,6 +119,7 @@ def make_block(self, category, parent_block, **kwargs):
         extra.update(kwargs)
         return ItemFactory.create(
             category=category,
+            parent=parent_block,
             parent_location=parent_block.location,
             modulestore=self.store,
             **extra