Skip to content

Add support for Import and Export of Libraries #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Kelketek wants to merge 25 commits into from
Closed

Add support for Import and Export of Libraries #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
73e2cb1
Studio support for creating and editing libraries (PR 6046)
bradenmacdonald Oct 29, 2014
6963e8e
Removed the ability to add Discussion and advanced components to Cont…
Kelketek Dec 2, 2014
459b651
Paging for LibraryView added with JS tests.
e-kolpakov Nov 3, 2014
523e8d9
Added tests for Library pagination.
Kelketek Dec 4, 2014
26e9399
Addressed notes from reviewers on Library Pagination.
Kelketek Dec 8, 2014
1581a39
Factored out Pagination into its own Container view.
Kelketek Dec 11, 2014
be3371e
Addressed further review notes for Library Pagination
Kelketek Dec 12, 2014
fcc2045
Merge pull request #6163 from open-craft/content_libraries/4-pagination
antoviaque Dec 22, 2014
219eeb6
Library Content XModule
bradenmacdonald Oct 29, 2014
d2ae624
Unit and integration tests of content libraries
bradenmacdonald Nov 2, 2014
d1681d0
LibraryContent bok choy acceptance tests
e-kolpakov Dec 8, 2014
e312c15
Friendly error message when library key is invalid
bradenmacdonald Dec 10, 2014
eecc6f1
Added explanation to container view of Library Block.
Kelketek Nov 26, 2014
a7b577d
Made errors on Library blocks use validate functionality.
Kelketek Nov 26, 2014
dfd0d70
Move update link to the validation area
bradenmacdonald Dec 10, 2014
e20b904
Fix: don't need to reload the whole page to refresh_children from the…
bradenmacdonald Dec 11, 2014
66f5f8f
Refresh children automatically when library setting is changed
bradenmacdonald Dec 11, 2014
c2f757f
Fix greedy intrusion of split_test documentation
bradenmacdonald Dec 11, 2014
82876da
Validation should be on the descriptor, not the module
bradenmacdonald Dec 14, 2014
c7c1b4c
ungettext fix
bradenmacdonald Dec 14, 2014
0537345
Move code that uses the modulestore to an XBlock service
bradenmacdonald Dec 14, 2014
262c875
Display the library key on the library view in Studio
bradenmacdonald Dec 19, 2014
0cc3b33
Remove state summary from the unit author page until SOL-192
bradenmacdonald Dec 19, 2014
16809dd
Merge pull request #6155 from open-craft/content_libraries/3-library-…
bradenmacdonald Dec 23, 2014
29b11e8
Added library import and export via .tar.gz'd XML files.
Kelketek Nov 12, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cms/djangoapps/contentstore/git_export_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_exporter import export_course_to_xml

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -129,7 +129,7 @@ def export_to_git(course_id, repo, user='', rdir=None):
root_dir = os.path.dirname(rdirp)
course_dir = os.path.basename(rdirp).rsplit('.git', 1)[0]
try:
export_to_xml(modulestore(), contentstore(), course_id,
export_course_to_xml(modulestore(), contentstore(), course_id,
root_dir, course_dir)
except (EnvironmentError, AttributeError):
log.exception('Failed export to xml')
Expand Down
4 changes: 2 additions & 2 deletions cms/djangoapps/contentstore/management/commands/export.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import os

from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_exporter import export_course_to_xml
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.keys import CourseKey
from xmodule.contentstore.django import contentstore
Expand Down Expand Up @@ -35,4 +35,4 @@ def handle(self, *args, **options):
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]

export_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
export_course_to_xml(modulestore(), contentstore(), course_key, root_dir, course_dir)
4 changes: 2 additions & 2 deletions cms/djangoapps/contentstore/management/commands/export_all_courses.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Script for exporting all courseware from Mongo to a directory and listing the courses which failed to export
"""
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_exporter import export_course_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore

Expand Down Expand Up @@ -49,7 +49,7 @@ def export_courses_to_output_path(output_path):
print(u"Exporting course id = {0} to {1}".format(course_id, output_path))
try:
course_dir = course_id.to_deprecated_string().replace('/', '...')
export_to_xml(module_store, content_store, course_id, root_dir, course_dir)
export_course_to_xml(module_store, content_store, course_id, root_dir, course_dir)
except Exception as err: # pylint: disable=broad-except
failed_export_courses.append(unicode(course_id))
print(u"=" * 30 + u"> Oops, failed to export {0}".format(course_id))
Expand Down
6 changes: 3 additions & 3 deletions cms/djangoapps/contentstore/management/commands/import.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from django.core.management.base import BaseCommand, CommandError, make_option
from django_comment_common.utils import (seed_permissions_roles,
are_permissions_roles_seeded)
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_importer import import_course_from_xml
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
Expand Down Expand Up @@ -40,11 +40,11 @@ def handle(self, *args, **options):
dis=do_import_static))
mstore = modulestore()

course_items = import_from_xml(
course_items = import_course_from_xml(
mstore, ModuleStoreEnum.UserID.mgmt_command, data_dir, course_dirs, load_error_modules=False,
static_content_store=contentstore(), verbose=True,
do_import_static=do_import_static,
create_course_if_not_present=True,
create_if_not_present=True,
)

for course in course_items:
Expand Down
4 changes: 2 additions & 2 deletions cms/djangoapps/contentstore/management/commands/tests/test_cleanup_assets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.mongo.base import location_to_query
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_from_xml
from xmodule.modulestore.xml_importer import import_course_from_xml
from django.conf import settings

TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
Expand All @@ -30,7 +30,7 @@ def test_export_all_courses(self):
This test validates that redundant Mac metadata files ('._example.txt', '.DS_Store') are
cleaned up on import
"""
import_from_xml(
import_course_from_xml(
self.module_store,
'**replace_user**',
TEST_DATA_DIR,
Expand Down
70 changes: 35 additions & 35 deletions cms/djangoapps/contentstore/tests/test_contentstore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
from opaque_keys.edx.keys import UsageKey, CourseKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation, CourseLocator
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, check_mongo_calls
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint
from xmodule.modulestore.xml_exporter import export_course_to_xml
from xmodule.modulestore.xml_importer import import_course_from_xml, perform_xlint

from xmodule.capa_module import CapaDescriptor
from xmodule.course_module import CourseDescriptor, Textbook
Expand Down Expand Up @@ -92,7 +92,7 @@ class ImportRequiredTestCases(ContentStoreTestCase):
Tests which legitimately need to import a course
"""
def test_no_static_link_rewrites_on_import(self):
course_items = import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_items = import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course = course_items[0]

handouts_usage_key = course.id.make_usage_key('course_info', 'handouts')
Expand All @@ -113,7 +113,7 @@ def test_about_overrides(self):
e.g. /about/Fall_2012/effort.html
while there is a base definition in /about/effort.html
'''
course_items = import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_items = import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_key = course_items[0].id
effort = self.store.get_item(course_key.make_usage_key('about', 'effort'))
self.assertEqual(effort.data, '6 hours')
Expand All @@ -129,7 +129,7 @@ def test_asset_import(self):
'''
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store, verbose=True)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store, verbose=True)

course = self.store.get_course(SlashSeparatedCourseKey('edX', 'toy', '2012_Fall'))

Expand Down Expand Up @@ -157,7 +157,7 @@ def test_course_info_updates_import_export(self):
"""
content_store = contentstore()
data_dir = TEST_DATA_DIR
courses = import_from_xml(
courses = import_course_from_xml(
self.store, self.user.id, data_dir, ['course_info_updates'],
static_content_store=content_store, verbose=True,
)
Expand Down Expand Up @@ -188,7 +188,7 @@ def test_course_info_updates_import_export(self):
# with same content as in course 'info' directory
root_dir = path(mkdtemp_clean())
print 'Exporting to tempdir = {0}'.format(root_dir)
export_to_xml(self.store, content_store, course.id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course.id, root_dir, 'test_export')

# check that exported course has files 'updates.html' and 'updates.items.json'
filesystem = OSFS(root_dir / 'test_export/info')
Expand All @@ -207,7 +207,7 @@ def test_course_info_updates_import_export(self):
def test_rewrite_nonportable_links_on_import(self):
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], static_content_store=content_store)

# first check a static asset link
course_key = SlashSeparatedCourseKey('edX', 'toy', 'run')
Expand Down Expand Up @@ -245,7 +245,7 @@ def test_export_course_roundtrip(self, mock_get):
print 'Exporting to tempdir = {0}'.format(root_dir)

# export out to a tempdir
export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')

# check for static tabs
self.verify_content_existence(self.store, root_dir, course_id, 'tabs', 'static_tab', '.html')
Expand Down Expand Up @@ -298,7 +298,7 @@ def test_export_course_roundtrip(self, mock_get):
def check_import(self, root_dir, content_store, course_id):
"""Imports the course in root_dir into the given course_id and verifies its content"""
# reimport
import_from_xml(
import_course_from_xml(
self.store,
self.user.id,
root_dir,
Expand Down Expand Up @@ -328,7 +328,7 @@ def verify_export_attrs_removed(attributes):
def test_export_course_with_metadata_only_video(self):
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

# create a new video module and add it as a child to a vertical
Expand All @@ -347,7 +347,7 @@ def test_export_course_with_metadata_only_video(self):
print 'Exporting to tempdir = {0}'.format(root_dir)

# export out to a tempdir
export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')

shutil.rmtree(root_dir)

Expand All @@ -357,7 +357,7 @@ def test_export_course_with_metadata_only_word_cloud(self):
"""
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['word_cloud'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['word_cloud'])
course_id = SlashSeparatedCourseKey('HarvardX', 'ER22x', '2013_Spring')

verticals = self.store.get_items(course_id, qualifiers={'category': 'vertical'})
Expand All @@ -373,7 +373,7 @@ def test_export_course_with_metadata_only_word_cloud(self):
print 'Exporting to tempdir = {0}'.format(root_dir)

# export out to a tempdir
export_to_xml(self.store, content_store, course_id, root_dir, 'test_export')
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_export')

shutil.rmtree(root_dir)

Expand All @@ -384,7 +384,7 @@ def test_empty_data_roundtrip(self):
"""
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

verticals = self.store.get_items(course_id, qualifiers={'category': 'vertical'})
Expand All @@ -400,10 +400,10 @@ def test_empty_data_roundtrip(self):

# Export the course
root_dir = path(mkdtemp_clean())
export_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')

# Reimport and get the video back
import_from_xml(self.store, self.user.id, root_dir)
import_course_from_xml(self.store, self.user.id, root_dir)
imported_word_cloud = self.store.get_item(course_id.make_usage_key('word_cloud', 'untitled'))

# It should now contain empty data
Expand All @@ -415,16 +415,16 @@ def test_html_export_roundtrip(self):
"""
content_store = contentstore()

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])

course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')

# Export the course
root_dir = path(mkdtemp_clean())
export_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')
export_course_to_xml(self.store, content_store, course_id, root_dir, 'test_roundtrip')

# Reimport and get the video back
import_from_xml(self.store, self.user.id, root_dir)
import_course_from_xml(self.store, self.user.id, root_dir)

# get the sample HTML with styling information
html_module = self.store.get_item(course_id.make_usage_key('html', 'with_styling'))
Expand All @@ -437,19 +437,19 @@ def test_html_export_roundtrip(self):
def test_export_course_without_content_store(self):
# Create toy course

course_items = import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_items = import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_id = course_items[0].id

root_dir = path(mkdtemp_clean())

print 'Exporting to tempdir = {0}'.format(root_dir)
export_to_xml(self.store, None, course_id, root_dir, 'test_export_no_content_store')
export_course_to_xml(self.store, None, course_id, root_dir, 'test_export_no_content_store')

# Delete the course from module store and reimport it

self.store.delete_course(course_id, self.user.id)

import_from_xml(
import_course_from_xml(
self.store, self.user.id, root_dir, ['test_export_no_content_store'],
static_content_store=None,
target_course_id=course_id
Expand All @@ -472,7 +472,7 @@ def test_export_course_no_xml_attributes(self):
exported successfully
"""
content_store = contentstore()
import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_id = SlashSeparatedCourseKey('edX', 'toy', '2012_Fall')
verticals = self.store.get_items(course_id, qualifiers={'category': 'vertical'})
vertical = verticals[0]
Expand All @@ -493,7 +493,7 @@ def test_export_course_no_xml_attributes(self):

# export should still complete successfully
root_dir = path(mkdtemp_clean())
export_to_xml(
export_course_to_xml(
self.store,
content_store,
course_id,
Expand Down Expand Up @@ -1261,16 +1261,16 @@ def test_get_html(handler):
)
self.assertEqual(resp.status_code, 200)

course_items = import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['simple'])
course_items = import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['simple'])
course_key = course_items[0].id

resp = self._show_course_overview(course_key)
self.assertEqual(resp.status_code, 200)
self.assertContains(resp, 'Chapter 2')

# go to various pages
test_get_html('import_handler')
test_get_html('export_handler')
test_get_html('course_import_handler')
test_get_html('course_export_handler')
test_get_html('course_team_handler')
test_get_html('course_info_handler')
test_get_html('checklists_handler')
Expand Down Expand Up @@ -1308,7 +1308,7 @@ def test_import_into_new_course_id(self):
target_course_id = _get_course_id(self.course_data)
_create_course(self, target_course_id, self.course_data)

import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)

modules = self.store.get_items(target_course_id)

Expand Down Expand Up @@ -1343,7 +1343,7 @@ def test_import_into_new_course_id_wiki_slug_renamespacing(self):
course_module.save()

# Import a course with wiki_slug == location.course
import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)
course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'toy')

Expand All @@ -1358,17 +1358,17 @@ def test_import_into_new_course_id_wiki_slug_renamespacing(self):
_create_course(self, target_course_id, course_data)

# Import a course with wiki_slug == location.course
import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'], target_course_id=target_course_id)
course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.111.2013_Spring')

# Now try importing a course with wiki_slug == '{0}.{1}.{2}'.format(location.org, location.course, location.run)
import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['two_toys'], target_course_id=target_course_id)
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['two_toys'], target_course_id=target_course_id)
course_module = self.store.get_course(target_course_id)
self.assertEquals(course_module.wiki_slug, 'MITx.111.2013_Spring')

def test_import_metadata_with_attempts_empty_string(self):
import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['simple'])
import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['simple'])
did_load_item = False
try:
course_key = SlashSeparatedCourseKey('edX', 'simple', 'problem')
Expand All @@ -1390,7 +1390,7 @@ def test_forum_id_generation(self):
self.assertNotEquals(new_discussion_item.discussion_id, '$$GUID$$')

def test_metadata_inheritance(self):
course_items = import_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])
course_items = import_course_from_xml(self.store, self.user.id, TEST_DATA_DIR, ['toy'])

course = course_items[0]
verticals = self.store.get_items(course.id, qualifiers={'category': 'vertical'})
Expand Down Expand Up @@ -1456,7 +1456,7 @@ def test_image_import(self):
content_store = contentstore()

# Use conditional_and_poll, as it's got an image already
courses = import_from_xml(
courses = import_course_from_xml(
self.store,
self.user.id,
TEST_DATA_DIR,
Expand Down
Loading