We change the data as well to check that non-overriden fields do get updated.
"
+ self.problem.data = new_data_value
+ modulestore().update_item(self.problem, self.user.id)
+
+ self.lc_block = self._refresh_children(self.lc_block)
+ self.problem_in_course = modulestore().get_item(self.problem_in_course.location)
+
+ self.assertEqual(self.problem_in_course.display_name, new_display_name)
+ self.assertEqual(self.problem_in_course.weight, new_weight)
+ self.assertEqual(self.problem_in_course.data, new_data_value)
diff --git a/cms/djangoapps/contentstore/views/item.py b/cms/djangoapps/contentstore/views/item.py
index 32d3fc49b269..773bcf18c75b 100644
--- a/cms/djangoapps/contentstore/views/item.py
+++ b/cms/djangoapps/contentstore/views/item.py
@@ -559,7 +559,10 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, user, display_
category = dest_usage_key.block_type
# Update the display name to indicate this is a duplicate (unless display name provided).
- duplicate_metadata = own_metadata(source_item)
+ duplicate_metadata = {} # Can't use own_metadata(), b/c it converts data for JSON serialization - not suitable for setting metadata of the new block
+ for field in source_item.fields.values():
+ if (field.scope == Scope.settings and field.is_set_on(source_item)):
+ duplicate_metadata[field.name] = field.read_from(source_item)
if display_name is not None:
duplicate_metadata['display_name'] = display_name
else:
@@ -584,7 +587,8 @@ def _duplicate_item(parent_usage_key, duplicate_source_usage_key, user, display_
dest_module.children = []
for child in source_item.children:
dupe = _duplicate_item(dest_module.location, child, user=user)
- dest_module.children.append(dupe)
+ if dupe not in dest_module.children: # _duplicate_item may add the child for us.
+ dest_module.children.append(dupe)
store.update_item(dest_module, user.id)
if 'detached' not in source_item.runtime.load_block_type(category)._class_tags:
diff --git a/common/lib/xmodule/xmodule/library_content_module.py b/common/lib/xmodule/xmodule/library_content_module.py
index 1b68e27a59c2..9cedc5bbff40 100644
--- a/common/lib/xmodule/xmodule/library_content_module.py
+++ b/common/lib/xmodule/xmodule/library_content_module.py
@@ -323,7 +323,7 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe
js_module_name = "VerticalDescriptor"
@XBlock.handler
- def refresh_children(self, request=None, suffix=None, update_db=True): # pylint: disable=unused-argument
+ def refresh_children(self, request=None, suffix=None): # pylint: disable=unused-argument
"""
Refresh children:
This method is to be used when any of the libraries that this block
@@ -335,15 +335,12 @@ def refresh_children(self, request=None, suffix=None, update_db=True): # pylint
This method will update this block's 'source_libraries' field to store
the version number of the libraries used, so we easily determine if
this block is up to date or not.
-
- If update_db is True (default), this will explicitly persist the changes
- to the modulestore by calling update_item()
"""
lib_tools = self.runtime.service(self, 'library_tools')
user_service = self.runtime.service(self, 'user')
user_perms = self.runtime.service(self, 'studio_user_permissions')
user_id = user_service.user_id if user_service else None # May be None when creating bok choy test fixtures
- lib_tools.update_children(self, user_id, user_perms, update_db)
+ lib_tools.update_children(self, user_id, user_perms)
return Response()
def _validate_library_version(self, validation, lib_tools, version, library_key):
@@ -451,7 +448,7 @@ def editor_saved(self, user, old_metadata, old_content):
if (set(old_source_libraries) != set(self.source_libraries) or
old_metadata.get('capa_type', ANY_CAPA_TYPE_VALUE) != self.capa_type):
try:
- self.refresh_children(None, None, update_db=False) # update_db=False since update_item() is about to be called anyways
+ self.refresh_children()
except ValueError:
pass # The validation area will display an error message, no need to do anything now.
diff --git a/common/lib/xmodule/xmodule/library_tools.py b/common/lib/xmodule/xmodule/library_tools.py
index f9b9f3edab83..4bdf51bdce53 100644
--- a/common/lib/xmodule/xmodule/library_tools.py
+++ b/common/lib/xmodule/xmodule/library_tools.py
@@ -1,10 +1,8 @@
"""
XBlock runtime services for LibraryContentModule
"""
-import hashlib
from django.core.exceptions import PermissionDenied
from opaque_keys.edx.locator import LibraryLocator
-from xblock.fields import Scope
from xmodule.library_content_module import LibraryVersionReference, ANY_CAPA_TYPE_VALUE
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.capa_module import CapaDescriptor
@@ -60,7 +58,7 @@ def _filter_child(self, usage_key, capa_type):
assert isinstance(descriptor, CapaDescriptor)
return capa_type in descriptor.problem_types
- def update_children(self, dest_block, user_id, user_perms=None, update_db=True):
+ def update_children(self, dest_block, user_id, user_perms=None):
"""
This method is to be used when any of the libraries that a LibraryContentModule
references have been updated. It will re-fetch all matching blocks from
@@ -71,82 +69,28 @@ def update_children(self, dest_block, user_id, user_perms=None, update_db=True):
This method will update dest_block's 'source_libraries' field to store
the version number of the libraries used, so we easily determine if
dest_block is up to date or not.
-
- If update_db is True (default), this will explicitly persist the changes
- to the modulestore by calling update_item(). Only set update_db False if
- you know for sure that dest_block is about to be saved to the modulestore
- anyways. Otherwise, orphaned blocks may be created.
"""
- root_children = []
if user_perms and not user_perms.can_write(dest_block.location.course_key):
raise PermissionDenied()
- with self.store.bulk_operations(dest_block.location.course_key):
- # Currently, ALL children are essentially deleted and then re-added
- # in a way that preserves their block_ids (and thus should preserve
- # student data, grades, analytics, etc.)
- # Once course-level field overrides are implemented, this will
- # change to a more conservative implementation.
-
- # First, load and validate the source_libraries:
- libraries = []
- for library_key, old_version in dest_block.source_libraries: # pylint: disable=unused-variable
- library = self._get_library(library_key)
- if library is None:
- raise ValueError("Required library not found.")
- if user_perms and not user_perms.can_read(library_key):
- raise PermissionDenied()
- libraries.append((library_key, library))
+ new_libraries = []
+ source_blocks = []
+ for library_key, __ in dest_block.source_libraries:
+ library = self._get_library(library_key)
+ if library is None:
+ raise ValueError("Required library not found.")
+ if user_perms and not user_perms.can_read(library_key):
+ raise PermissionDenied()
+ filter_children = (dest_block.capa_type != ANY_CAPA_TYPE_VALUE)
+ if filter_children:
+ # Apply simple filtering based on CAPA problem types:
+ source_blocks.extend([key for key in library.children if self._filter_child(key, dest_block.capa_type)])
+ else:
+ source_blocks.extend(library.children)
+ new_libraries.append(LibraryVersionReference(library_key, library.location.library_key.version_guid))
- # Next, delete all our existing children to avoid block_id conflicts when we add them:
- for child in dest_block.children:
- self.store.delete_item(child, user_id)
-
- # Now add all matching children, and record the library version we use:
- new_libraries = []
- for library_key, library in libraries:
-
- def copy_children_recursively(from_block, filter_problem_type=False):
- """
- Internal method to copy blocks from the library recursively
- """
- new_children = []
- if filter_problem_type:
- filtered_children = [key for key in from_block.children if self._filter_child(key, dest_block.capa_type)]
- else:
- filtered_children = from_block.children
- for child_key in filtered_children:
- child = self.store.get_item(child_key, depth=None)
- # We compute a block_id for each matching child block found in the library.
- # block_ids are unique within any branch, but are not unique per-course or globally.
- # We need our block_ids to be consistent when content in the library is updated, so
- # we compute block_id as a hash of three pieces of data:
- unique_data = "{}:{}:{}".format(
- dest_block.location.block_id, # Must not clash with other usages of the same library in this course
- unicode(library_key.for_version(None)).encode("utf-8"), # The block ID below is only unique within a library, so we need this too
- child_key.block_id, # Child block ID. Should not change even if the block is edited.
- )
- child_block_id = hashlib.sha1(unique_data).hexdigest()[:20]
- fields = {}
- for field in child.fields.itervalues():
- if field.scope == Scope.settings and field.is_set_on(child):
- fields[field.name] = field.read_from(child)
- if child.has_children:
- fields['children'] = copy_children_recursively(from_block=child)
- new_child_info = self.store.create_item(
- user_id,
- dest_block.location.course_key,
- child_key.block_type,
- block_id=child_block_id,
- definition_locator=child.definition_locator,
- runtime=dest_block.system,
- fields=fields,
- )
- new_children.append(new_child_info.location)
- return new_children
- root_children.extend(copy_children_recursively(from_block=library, filter_problem_type=True))
- new_libraries.append(LibraryVersionReference(library_key, library.location.library_key.version_guid))
+ with self.store.bulk_operations(dest_block.location.course_key):
dest_block.source_libraries = new_libraries
- dest_block.children = root_children
- if update_db:
- self.store.update_item(dest_block, user_id)
+ self.store.update_item(dest_block, user_id)
+ dest_block.children = self.store.copy_from_template(source_blocks, dest_block.location, user_id)
+ # ^-- copy_from_template updates the children in the DB but we must also set .children here to avoid overwriting the DB again
diff --git a/common/lib/xmodule/xmodule/modulestore/inheritance.py b/common/lib/xmodule/xmodule/modulestore/inheritance.py
index 296fdb80caa6..3ec2f96dbd14 100644
--- a/common/lib/xmodule/xmodule/modulestore/inheritance.py
+++ b/common/lib/xmodule/xmodule/modulestore/inheritance.py
@@ -211,8 +211,8 @@ def inherit_metadata(descriptor, inherited_data):
def own_metadata(module):
"""
- Return a dictionary that contains only non-inherited field keys,
- mapped to their serialized values
+ Return a JSON-friendly dictionary that contains only non-inherited field
+ keys, mapped to their serialized values
"""
return module.get_explicitly_set_fields_by_scope(Scope.settings)
@@ -283,6 +283,8 @@ def has(self, key):
def default(self, key):
"""
- Check to see if the default should be from inheritance rather than from the field's global default
+ Check to see if the default should be from inheritance. If not
+ inheriting, this will raise KeyError which will cause the caller to use
+ the field's global default.
"""
return self.inherited_settings[key.field_name]
diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py
index 24e68ea14308..617542765975 100644
--- a/common/lib/xmodule/xmodule/modulestore/mixed.py
+++ b/common/lib/xmodule/xmodule/modulestore/mixed.py
@@ -676,6 +676,14 @@ def import_xblock(self, user_id, course_key, block_type, block_id, fields=None,
store = self._verify_modulestore_support(course_key, 'import_xblock')
return store.import_xblock(user_id, course_key, block_type, block_id, fields, runtime)
+ @strip_key
+ def copy_from_template(self, source_keys, dest_key, user_id, **kwargs):
+ """
+ See :py:meth `SplitMongoModuleStore.copy_from_template`
+ """
+ store = self._verify_modulestore_support(dest_key.course_key, 'copy_from_template')
+ return store.copy_from_template(source_keys, dest_key, user_id)
+
@strip_key
def update_item(self, xblock, user_id, allow_not_found=False, **kwargs):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
index 3c357c87517e..73a950b3558d 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/caching_descriptor_system.py
@@ -169,16 +169,17 @@ def xblock_from_json(
if block_key is None:
block_key = BlockKey(json_data['block_type'], LocalId())
+ convert_fields = lambda field: self.modulestore.convert_references_to_keys(
+ course_key, class_, field, self.course_entry.structure['blocks'],
+ )
+
if definition_id is not None and not json_data.get('definition_loaded', False):
definition_loader = DefinitionLazyLoader(
self.modulestore,
course_key,
block_key.type,
definition_id,
- lambda fields: self.modulestore.convert_references_to_keys(
- course_key, self.load_block_type(block_key.type),
- fields, self.course_entry.structure['blocks'],
- )
+ convert_fields,
)
else:
definition_loader = None
@@ -193,9 +194,8 @@ def xblock_from_json(
block_id=block_key.id,
)
- converted_fields = self.modulestore.convert_references_to_keys(
- block_locator.course_key, class_, json_data.get('fields', {}), self.course_entry.structure['blocks'],
- )
+ converted_fields = convert_fields(json_data.get('fields', {}))
+ converted_defaults = convert_fields(json_data.get('defaults', {}))
if block_key in self._parent_map:
parent_key = self._parent_map[block_key]
parent = course_key.make_usage_key(parent_key.type, parent_key.id)
@@ -204,6 +204,7 @@ def xblock_from_json(
kvs = SplitMongoKVS(
definition_loader,
converted_fields,
+ converted_defaults,
parent=parent,
field_decorator=kwargs.get('field_decorator')
)
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
index 8b4b872c4301..9535a5232d01 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -255,7 +255,7 @@ def get_definitions(self, definitions):
"""
Retrieve all definitions listed in `definitions`.
"""
- return self.definitions.find({'$in': {'_id': definitions}})
+ return self.definitions.find({'_id': {'$in': definitions}})
def insert_definition(self, definition):
"""
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
index 5cb62bcf57be..6c375d2dfc02 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
@@ -27,6 +27,8 @@
**** 'definition': the db id of the record containing the content payload for this xblock
**** 'fields': the Scope.settings and children field values
***** 'children': This is stored as a list of (block_type, block_id) pairs
+ **** 'defaults': Scope.settings default values copied from a template block (used e.g. when
+ blocks are copied from a library to a course)
**** 'edit_info': dictionary:
***** 'edited_on': when was this xblock's fields last changed (will be edited_on value of
update_version structure)
@@ -53,6 +55,7 @@
import copy
import threading
import datetime
+import hashlib
import logging
from contracts import contract, new_contract
from importlib import import_module
@@ -670,7 +673,7 @@ def cache_items(self, system, base_block_ids, course_key, depth=0, lazy=True):
new_module_data = {}
for block_id in base_block_ids:
new_module_data = self.descendants(
- system.course_entry.structure['blocks'],
+ copy.deepcopy(system.course_entry.structure['blocks']), # copy or our changes like setting 'definition_loaded' will affect the active bulk operation data
block_id,
depth,
new_module_data
@@ -691,12 +694,9 @@ def cache_items(self, system, base_block_ids, course_key, depth=0, lazy=True):
for block in new_module_data.itervalues():
if block['definition'] in definitions:
- converted_fields = self.convert_references_to_keys(
- course_key, system.load_block_type(block['block_type']),
- definitions[block['definition']].get('fields'),
- system.course_entry.structure['blocks'],
- )
- block['fields'].update(converted_fields)
+ definition = definitions[block['definition']]
+ # convert_fields was being done here, but it gets done later in the runtime's xblock_from_json
+ block['fields'].update(definition.get('fields'))
block['definition_loaded'] = True
system.module_data.update(new_module_data)
@@ -2071,6 +2071,154 @@ def copy(self, user_id, source_course, destination_course, subtree_list=None, bl
self.update_structure(destination_course, destination_structure)
self._update_head(destination_course, index_entry, destination_course.branch, destination_structure['_id'])
+ @contract(source_keys="list(BlockUsageLocator)", dest_usage=BlockUsageLocator)
+ def copy_from_template(self, source_keys, dest_usage, user_id):
+ """
+ Flexible mechanism for inheriting content from an external course/library/etc.
+
+ Will copy all of the XBlocks whose keys are passed as `source_course` so that they become
+ children of the XBlock whose key is `dest_usage`. Any previously existing children of
+ `dest_usage` that haven't been replaced/updated by this copy_from_template operation will
+ be deleted.
+
+ Unlike `copy()`, this does not care whether the resulting blocks are positioned similarly
+ in their new course/library. However, the resulting blocks will be in the same relative
+ order as `source_keys`.
+
+ If any of the blocks specified already exist as children of the destination block, they
+ will be updated rather than duplicated or replaced. If they have Scope.settings field values
+ overriding inherited default values, those overrides will be preserved.
+
+ IMPORTANT: This method does not preserve block_id - in other words, every block that is
+ copied will be assigned a new block_id. This is because we assume that the same source block
+ may be copied into one course in multiple places. However, it *is* guaranteed that every
+ time this method is called for the same source block and dest_usage, the same resulting
+ block id will be generated.
+
+ :param source_keys: a list of BlockUsageLocators. Order is preserved.
+
+ :param dest_usage: The BlockUsageLocator that will become the parent of an inherited copy
+ of all the xblocks passed in `source_keys`.
+
+ :param user_id: The user who will get credit for making this change.
+ """
+ # Preload the block structures for all source courses/libraries/etc.
+ # so that we can access descendant information quickly
+ source_structures = {}
+ for key in source_keys:
+ course_key = key.course_key.for_version(None)
+ if course_key.branch is None:
+ raise ItemNotFoundError("branch is required for all source keys when using copy_from_template")
+ if course_key not in source_structures:
+ with self.bulk_operations(course_key):
+ source_structures[course_key] = self._lookup_course(course_key).structure
+
+ destination_course = dest_usage.course_key
+ with self.bulk_operations(destination_course):
+ index_entry = self.get_course_index(destination_course)
+ if index_entry is None:
+ raise ItemNotFoundError(destination_course)
+ dest_structure = self._lookup_course(destination_course).structure
+ old_dest_structure_version = dest_structure['_id']
+ dest_structure = self.version_structure(destination_course, dest_structure, user_id)
+
+ # Set of all descendent block IDs of dest_usage that are to be replaced:
+ block_key = BlockKey(dest_usage.block_type, dest_usage.block_id)
+ orig_descendants = set(self.descendants(dest_structure['blocks'], block_key, depth=None, descendent_map={}))
+ orig_descendants.remove(block_key) # The descendants() method used above adds the block itself, which we don't consider a descendant.
+ new_descendants = self._copy_from_template(source_structures, source_keys, dest_structure, block_key, user_id)
+
+ # Update the edit info:
+ dest_info = dest_structure['blocks'][block_key]
+
+ # Update the edit_info:
+ dest_info['edit_info']['previous_version'] = dest_info['edit_info']['update_version']
+ dest_info['edit_info']['update_version'] = old_dest_structure_version
+ dest_info['edit_info']['edited_by'] = user_id
+ dest_info['edit_info']['edited_on'] = datetime.datetime.now(UTC)
+
+ orphans = orig_descendants - new_descendants
+ for orphan in orphans:
+ del dest_structure['blocks'][orphan]
+
+ self.update_structure(destination_course, dest_structure)
+ self._update_head(destination_course, index_entry, destination_course.branch, dest_structure['_id'])
+ # Return usage locators for all the new children:
+ return [destination_course.make_usage_key(*k) for k in dest_structure['blocks'][block_key]['fields']['children']]
+
+ def _copy_from_template(self, source_structures, source_keys, dest_structure, new_parent_block_key, user_id):
+ """
+ Internal recursive implementation of copy_from_template()
+
+ Returns the new set of BlockKeys that are the new descendants of the block with key 'block_key'
+ """
+ # pylint: disable=no-member
+ # ^-- Until pylint gets namedtuple support, it will give warnings about BlockKey attributes
+ new_blocks = set()
+
+ new_children = list() # ordered list of the new children of new_parent_block_key
+
+ for usage_key in source_keys:
+ src_course_key = usage_key.course_key.for_version(None)
+ block_key = BlockKey(usage_key.block_type, usage_key.block_id)
+ source_structure = source_structures.get(src_course_key, [])
+ if block_key not in source_structure['blocks']:
+ raise ItemNotFoundError(usage_key)
+ source_block_info = source_structure['blocks'][block_key]
+
+ # Compute a new block ID. This new block ID must be consistent when this
+ # method is called with the same (source_key, dest_structure) pair
+ unique_data = "{}:{}:{}".format(
+ unicode(src_course_key).encode("utf-8"),
+ block_key.id,
+ new_parent_block_key.id,
+ )
+ new_block_id = hashlib.sha1(unique_data).hexdigest()[:20]
+ new_block_key = BlockKey(block_key.type, new_block_id)
+
+ # Now clone block_key to new_block_key:
+ new_block_info = copy.deepcopy(source_block_info)
+ # Note that new_block_info now points to the same definition ID entry as source_block_info did
+ existing_block_info = dest_structure['blocks'].get(new_block_key, {})
+ # Inherit the Scope.settings values from 'fields' to 'defaults'
+ new_block_info['defaults'] = new_block_info['fields']
+
+ #