From 6078bbba19853478ebc7bb69de9c1291dfc03bd8 Mon Sep 17 00:00:00 2001 From: Andrii Date: Wed, 14 May 2025 22:52:09 +0300 Subject: [PATCH 01/40] feat: redesign import task + api/helpers tests adaptation --- .../import_from_modulestore/admin.py | 54 ++ cms/djangoapps/import_from_modulestore/api.py | 43 ++ .../import_from_modulestore/data.py | 69 +++ .../import_from_modulestore/helpers.py | 466 ++++++++++++++++++ ...created_remove_import_modified_and_more.py | 48 ++ .../import_from_modulestore/models.py | 154 ++++++ .../import_from_modulestore/tasks.py | 146 ++++++ .../tests/factories.py | 26 + .../import_from_modulestore/tests/test_api.py | 98 ++++ .../tests/test_helpers.py | 442 +++++++++++++++++ .../tests/test_tasks.py | 176 +++++++ 11 files changed, 1722 insertions(+) create mode 100644 cms/djangoapps/import_from_modulestore/admin.py create mode 100644 cms/djangoapps/import_from_modulestore/api.py create mode 100644 cms/djangoapps/import_from_modulestore/data.py create mode 100644 cms/djangoapps/import_from_modulestore/helpers.py create mode 100644 cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_created_remove_import_modified_and_more.py create mode 100644 cms/djangoapps/import_from_modulestore/models.py create mode 100644 cms/djangoapps/import_from_modulestore/tasks.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/factories.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_api.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_helpers.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_tasks.py diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py new file mode 100644 index 000000000000..61f90f5f7d48 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/admin.py @@ -0,0 +1,54 @@ +""" +This module contains the admin configuration for the Import model. +""" +from django.contrib import admin + +from .models import Import, PublishableEntityImport, PublishableEntityMapping + + +class ImportAdmin(admin.ModelAdmin): + """ + Admin configuration for the Import model. + """ + + list_display = ( + 'uuid', + 'created', + 'status', + 'source_key', + 'target_change', + ) + list_filter = ( + 'user_task_status__state', + ) + search_fields = ( + 'source_key', + 'target_change', + ) + + raw_id_fields = ('user',) + readonly_fields = ('user_task_status',) + + def uuid(self, obj): + """ + Returns the UUID of the import. + """ + return getattr(obj.user_task_status, 'uuid', None) + uuid.short_description = 'UUID' + + def created(self, obj): + """ + Returns the creation date of the import. + """ + return getattr(obj.user_task_status, 'created', None) + + def status(self, obj): + """ + Returns the status of the import. + """ + return getattr(obj.user_task_status, 'state', None) + + +admin.site.register(Import, ImportAdmin) +admin.site.register(PublishableEntityImport) +admin.site.register(PublishableEntityMapping) diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py new file mode 100644 index 000000000000..5c9bfc2aaec9 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/api.py @@ -0,0 +1,43 @@ +""" +API for course to library import. +""" +from typing import Sequence + +from opaque_keys.edx.keys import LearningContextKey, UsageKey + +from .helpers import cancel_incomplete_old_imports +from .models import Import as _Import +from .tasks import import_to_library_task +from .validators import validate_usage_keys_to_import + + +def import_to_library( + source_key: LearningContextKey, + usage_ids: Sequence[str | UsageKey], + target_learning_package_id: int, + user_id: int, + composition_level: str, + override: bool = False, +) -> _Import: + """ + Import staged content to a library from staged content. + """ + validate_usage_keys_to_import(usage_ids) + + import_from_modulestore = _Import.objects.create( + source_key=source_key, + user_id=user_id, + composition_level=composition_level, + override=override, + ) + cancel_incomplete_old_imports(import_from_modulestore) + + import_to_library_task.apply_async( + kwargs={ + 'import_pk': import_from_modulestore.pk, + 'usage_key_strings': usage_ids, + 'learning_package_id': target_learning_package_id, + 'user_id': user_id, + }, + ) + return import_from_modulestore diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py new file mode 100644 index 000000000000..b7303281a985 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/data.py @@ -0,0 +1,69 @@ +""" +This module contains the data models for the import_from_modulestore app. +""" +from collections import namedtuple +from enum import Enum +from openedx.core.djangoapps.content_libraries import api as content_libraries_api + +from django.utils.translation import gettext_lazy as _ + + +class ImportStatus(Enum): + """ + The status of this modulestore-to-learning-core import. + """ + + WAITNG_TO_STAGE = _('Waiting to stage content') + STAGING = _('Staging content for import') + STAGING_FAILED = _('Failed to stage content') + STAGED = _('Content is staged and ready for import') + IMPORTING = _('Importing staged content') + IMPORTING_FAILED = _('Failed to import staged content') + IMPORTED = _('Staged content imported successfully') + CANCELED = _('Import canceled') + + FAILED_STATUSES = [ + STAGING_FAILED, + IMPORTING_FAILED, + ] + + +class CompositionLevel(Enum): + """ + Enumeration of composition levels for course content. + Defines the different levels of composition for course content, + including chapters, sequentials, verticals, and xblocks. + It also categorizes these levels into complicated and flat + levels for easier processing. + """ + + CHAPTER = content_libraries_api.ContainerType.Section + SEQUENTIAL = content_libraries_api.ContainerType.Subsection + VERTICAL = content_libraries_api.ContainerType.Unit + COMPONENT = 'component' + OLX_COMPLEX_LEVELS = [ + VERTICAL.olx_tag, + SEQUENTIAL.olx_tag, + CHAPTER.olx_tag, + ] + + @classmethod + def values(cls): + """ + Returns all levels of composition levels. + """ + return [composition_level.value for composition_level in cls] + + @classmethod + def choices(cls): + """ + Returns all levels of composition levels as a list of tuples. + """ + return [ + (composition_level.value, composition_level.name) + for composition_level in cls + if not isinstance(composition_level.value, list) + ] + + +PublishableVersionWithMapping = namedtuple('PublishableVersionWithMapping', ['publishable_version', 'mapping']) diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py new file mode 100644 index 000000000000..60b978270c4c --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -0,0 +1,466 @@ +""" +Helper functions for importing course content into a library. +""" +from datetime import datetime, timezone +from functools import partial +import logging +import mimetypes +import os +import secrets +from typing import TYPE_CHECKING + +from django.db import transaction +from django.db.utils import IntegrityError +from lxml import etree + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import CourseLocator +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import Component, Container, ContainerVersion, PublishableEntity + +from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.django import modulestore + +from .data import CompositionLevel, ImportStatus, PublishableVersionWithMapping +from .models import Import, PublishableEntityMapping + +if TYPE_CHECKING: + from openedx_learning.apps.authoring_models import LearningPackage + from xblock.core import XBlock + + from openedx.core.djangoapps.content_staging.api import _StagedContent as StagedContent + + +log = logging.getLogger(__name__) +parser = etree.XMLParser(strip_cdata=False) + + +class ImportClient: + """ + Client for importing course content into a library. + + This class handles the import of course content from staged content into a + content library, creating the appropriate container hierarchy based on the + specified composition level. + """ + + # The create functions have different kwarg names for the child list, + # so we need to use partial to set the child list to empty. + CONTAINER_CREATORS_MAP: dict[str, partial] = { + api.ContainerType.Section.olx_tag: partial(authoring_api.create_section_and_version, subsections=[]), + api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_subsection_and_version, units=[]), + api.ContainerType.Unit.olx_tag: partial(authoring_api.create_unit_and_version, components=[]), + } + + CONTAINER_OVERRIDERS_MAP: dict[str, partial] = { + api.ContainerType.Section.olx_tag: partial(authoring_api.create_next_section_version, subsections=[]), + api.ContainerType.Subsection.olx_tag: partial(authoring_api.create_next_subsection_version, units=[]), + api.ContainerType.Unit.olx_tag: partial(authoring_api.create_next_unit_version, components=[]), + } + + def __init__( + self, + import_event: Import, + block_usage_key_to_import: str, + target_learning_package: 'LearningPackage', + staged_content: 'StagedContent', + composition_level: str, + override: bool = False, + ): + self.import_event = import_event + self.block_usage_key_to_import = block_usage_key_to_import + self.learning_package = target_learning_package + self.staged_content = staged_content + self.composition_level = composition_level + self.override = override + + self.user_id = import_event.user_id + self.content_library = target_learning_package.contentlibrary + self.library_key = self.content_library.library_key + self.parser = etree.XMLParser(strip_cdata=False) + + def import_from_staged_content(self) -> list[PublishableVersionWithMapping]: + """ + Import staged content into a library. + """ + node = etree.fromstring(self.staged_content.olx, parser=parser) + usage_key = UsageKey.from_string(self.block_usage_key_to_import) + block_to_import = get_node_for_usage_key(node, usage_key) + if block_to_import is None: + return [] + + return self._import_complicated_child(block_to_import, self.block_usage_key_to_import) + + def _process_import(self, usage_key_string, block_to_import) -> list[PublishableVersionWithMapping]: + """ + Process import of a block from staged content into a library. + + Imports a block and its children into the library based on the + composition level. It handles both simple and complicated blocks, creating + the necessary container hierarchy. + """ + usage_key = UsageKey.from_string(usage_key_string) + result = [] + + if block_to_import.tag not in CompositionLevel.OLX_COMPLEX_LEVELS.value: + return self._import_simple_block(block_to_import, usage_key) + + for child in block_to_import.getchildren(): + child_usage_key_string = get_usage_key_string_from_staged_content( + self.staged_content, child.get('url_name') + ) + if not child_usage_key_string: + continue + + result.extend(self._import_child_block(child, child_usage_key_string)) + + if self.composition_level == CompositionLevel.COMPONENT.value: + return [ + publishable_version_with_mapping for publishable_version_with_mapping in result + if not isinstance(publishable_version_with_mapping.publishable_version, ContainerVersion) + ] + return result + + def _import_simple_block(self, block_to_import, usage_key) -> list[PublishableVersionWithMapping]: + """ + Import a simple block into the library. + + Creates a block in the library from the staged content block. + It returns a list containing the created component version. + """ + publishable_version_with_mapping = self._create_block_in_library(block_to_import, usage_key) + return [publishable_version_with_mapping] if publishable_version_with_mapping else [] + + def _import_child_block(self, child, child_usage_key_string): + """ + Import a child block into the library. + + Determines whether the child block is simple or complicated and + delegates the import process to the appropriate helper method. + """ + child_usage_key = UsageKey.from_string(child_usage_key_string) + if child.tag in CompositionLevel.OLX_COMPLEX_LEVELS.value: + return self._import_complicated_child(child, child_usage_key_string) + else: + return self._import_simple_block(child, child_usage_key) + + def _import_complicated_child(self, child, child_usage_key_string): + """ + Import a complicated child block into the library. + + Handles the import of complicated child blocks, including creating + containers and updating components. + Returns a list containing the created container version. + """ + if not self._should_create_container(child.tag): + return self._process_import(child_usage_key_string, child) + + container_version_with_mapping = self.get_or_create_container( + child.tag, + child.get('url_name'), + child.get('display_name', child.tag), + child_usage_key_string, + ) + child_component_versions_with_mapping = self._process_import(child_usage_key_string, child) + child_component_versions = [ + child_component_version.publishable_version for child_component_version + in child_component_versions_with_mapping + ] + self._update_container_components(container_version_with_mapping.publishable_version, child_component_versions) + return [container_version_with_mapping] + child_component_versions_with_mapping + + def _should_create_container(self, container_type: str) -> bool: + """ + Determine if a new container should be created. + + Container type should be at a lower level than the current composition level. + """ + composition_hierarchy = CompositionLevel.OLX_COMPLEX_LEVELS.value + return ( + container_type in composition_hierarchy and + self.composition_level in composition_hierarchy and + composition_hierarchy.index(container_type) <= composition_hierarchy.index(self.composition_level) + ) + + def get_or_create_container( + self, + container_type: str, + key: str, + display_name: str, + block_usage_key_string: str + ) -> PublishableVersionWithMapping: + """ + Create a container of the specified type. + + Creates a container (e.g., chapter, sequential, vertical) in the + content library. + """ + try: + container_creator_func = self.CONTAINER_CREATORS_MAP[container_type] + container_override_func = self.CONTAINER_OVERRIDERS_MAP[container_type] + except KeyError as exc: + raise ValueError(f"Unknown container type: {container_type}") from exc + + try: + container_version = self.content_library.learning_package.publishable_entities.get(key=key) + except PublishableEntity.DoesNotExist: + container_version = None + + if container_version and self.override: + container_version = container_override_func( + container_version.container, + title=display_name or f"New {container_type}", + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + elif not container_version: + _, container_version = container_creator_func( + self.learning_package.id, + key=key or secrets.token_hex(16), + title=display_name or f"New {container_type}", + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + + publishable_entity_mapping, _ = get_or_create_publishable_entity_mapping( + block_usage_key_string, + container_version.container + ) + + return PublishableVersionWithMapping(container_version, publishable_entity_mapping) + + def _update_container_components(self, container_version, component_versions): + """ + Update components of a container. + """ + entity_rows = [ + authoring_api.ContainerEntityRow( + entity_pk=cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk, + version_pk=cv.pk, + ) + for cv in component_versions + ] + return authoring_api.create_next_container_version( + container_pk=container_version.container.pk, + title=container_version.title, + entity_rows=entity_rows, + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + container_version_cls=container_version.__class__, + ) + + def _create_block_in_library(self, block_to_import, usage_key) -> PublishableVersionWithMapping | None: + """ + Create a block in a library from a staged content block. + """ + now = datetime.now(tz=timezone.utc) + staged_content_files = content_staging_api.get_staged_content_static_files(self.staged_content.id) + + with transaction.atomic(): + component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) + does_component_exist = authoring_api.get_components( + self.learning_package.id + ).filter(local_key=usage_key.block_id).exists() + + if does_component_exist: + if not self.override: + log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.") + return None + else: + component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import)) + else: + try: + _, library_usage_key = api.validate_can_add_block_to_library( + self.library_key, + block_to_import.tag, + usage_key.block_id, + ) + except api.IncompatibleTypesError as e: + log.error(f"Error validating block {usage_key} for library {self.library_key}: {e}") + return None + + authoring_api.create_component( + self.learning_package.id, + component_type=component_type, + local_key=usage_key.block_id, + created=now, + created_by=self.import_event.user_id, + ) + component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) + + self._process_staged_content_files( + component_version, + staged_content_files, + usage_key, + block_to_import, + now, + ) + publishable_entity_mapping, _ = get_or_create_publishable_entity_mapping( + usage_key, + component_version.component + ) + return PublishableVersionWithMapping(component_version, publishable_entity_mapping) + + def _handle_component_override(self, usage_key, new_content): + """ + Create new ComponentVersion for overridden component. + """ + component_version = None + try: + component = authoring_api.get_components(self.learning_package.id).get(local_key=usage_key.block_id) + except Component.DoesNotExist: + return component_version + library_usage_key = api.library_component_usage_key(self.library_key, component) + + component_version = api.set_library_block_olx(library_usage_key, new_content) + + return component_version + + def _process_staged_content_files( + self, + component_version, + staged_content_files, + usage_key, + block_to_import, + created_at, + ): + """ + Process staged content files for a component. + + Processes the staged content files for a component, creating the + necessary file content and associating it with the component version. + """ + block_olx = etree.tostring(block_to_import).decode('utf-8') + + for staged_content_file_data in staged_content_files: + original_filename = staged_content_file_data.filename + file_basename = os.path.basename(original_filename) + file_basename_no_ext, _ = os.path.splitext(file_basename) + + # Skip files not referenced in the block + if file_basename not in block_olx and file_basename_no_ext not in block_olx: + log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") + continue + + file_data = content_staging_api.get_staged_content_static_file_data( + self.staged_content.id, + original_filename, + ) + if not file_data: + log.error( + f"Staged content {self.staged_content.id} included referenced " + f"file {original_filename}, but no file data was found." + ) + continue + + filename = f"static/{file_basename}" + media_type_str, _ = mimetypes.guess_type(filename) + if not media_type_str: + media_type_str = "application/octet-stream" + + media_type = authoring_api.get_or_create_media_type(media_type_str) + content = authoring_api.get_or_create_file_content( + self.learning_package.id, + media_type.id, + data=file_data, + created=created_at, + ) + + try: + authoring_api.create_component_version_content(component_version.pk, content.id, key=filename) + except IntegrityError: + pass # Content already exists + + +def import_from_staged_content( + import_event: Import, + usage_key_string: str, + target_learning_package: 'LearningPackage', + staged_content: 'StagedContent', + composition_level: str, + override: bool = False, +) -> list[PublishableVersionWithMapping]: + """ + Import staged content to a library from staged content. + + Returns a list of PublishableVersionWithMappings created during the import. + """ + import_client = ImportClient( + import_event, + usage_key_string, + target_learning_package, + staged_content, + composition_level, + override, + ) + return import_client.import_from_staged_content() + + +def get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]: + """ + Creates a mapping between the source usage key and the target publishable entity. + """ + if isinstance(component, Container): + target_package = component.publishable_entity.learning_package + else: + target_package = component.learning_package + return PublishableEntityMapping.objects.get_or_create( + source_usage_key=usage_key, + target_entity=component.publishable_entity, + target_package=target_package, + ) + + +def get_usage_key_string_from_staged_content(staged_content: 'StagedContent', block_id: str) -> str | None: + """ + Get the usage ID from a staged content by block ID. + """ + if staged_content.tags is None: + return None + return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None) + + +def get_node_for_usage_key(node: etree._Element, usage_key: UsageKey) -> etree._Element: + """ + Get the node in an XML tree which matches to the usage key. + """ + if node.tag == usage_key.block_type and node.get('url_name') == usage_key.block_id: + return node + + for child in node.getchildren(): + found = get_node_for_usage_key(child, usage_key) + if found is not None: + return found + + +def get_items_to_import(import_event: Import) -> list['XBlock']: + """ + Collect items to import from a course. + """ + items_to_import: list['XBlock'] = [] + if isinstance(import_event.source_key, CourseLocator): + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or [] + ) + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "static_tab"}) or [] + ) + + return items_to_import + + +def cancel_incomplete_old_imports(import_event: Import) -> None: + """ + Cancel any incomplete imports that have the same target as the current import. + + When a new import is created, we want to cancel any other incomplete user imports that have the same target. + """ + incomplete_user_imports_with_same_target = Import.objects.filter( + user=import_event.user, + target_change=import_event.target_change, + source_key=import_event.source_key, + staged_content_for_import__isnull=False + ).exclude(pk=import_event.pk) + for incomplete_import in incomplete_user_imports_with_same_target: + incomplete_import.set_status(ImportStatus.CANCELED) diff --git a/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_created_remove_import_modified_and_more.py b/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_created_remove_import_modified_and_more.py new file mode 100644 index 000000000000..96b2d6ff69e6 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/migrations/0002_remove_import_created_remove_import_modified_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.20 on 2025-05-14 17:31 + +from django.db import migrations, models +import django.db.models.deletion +from openedx.core.djangoapps.content_libraries.api import ContainerType + + +class Migration(migrations.Migration): + + dependencies = [ + ('user_tasks', '0004_url_textfield'), + ('import_from_modulestore', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='import', + name='created', + ), + migrations.RemoveField( + model_name='import', + name='modified', + ), + migrations.RemoveField( + model_name='import', + name='status', + ), + migrations.RemoveField( + model_name='import', + name='uuid', + ), + migrations.AddField( + model_name='import', + name='composition_level', + field=models.CharField(choices=[(ContainerType['Section'], 'CHAPTER'), (ContainerType['Subsection'], 'SEQUENTIAL'), (ContainerType['Unit'], 'VERTICAL'), ('component', 'COMPONENT')], default='component', help_text='The composition level of the target learning package', max_length=255), + preserve_default=False, + ), + migrations.AddField( + model_name='import', + name='override', + field=models.BooleanField(default=False, help_text='If true, the import will override any existing content in the target learning package.'), + ), + migrations.AddField( + model_name='import', + name='user_task_status', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='user_tasks.usertaskstatus'), + ), + ] diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py new file mode 100644 index 000000000000..e8e9fa4e3f96 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/models.py @@ -0,0 +1,154 @@ +""" +Models for the course to library import app. +""" + +from django.contrib.auth import get_user_model +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from model_utils.models import TimeStampedModel +from opaque_keys.edx.django.models import ( + LearningContextKeyField, + UsageKeyField, +) +from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity + +from .data import CompositionLevel, ImportStatus + +User = get_user_model() + + +class Import(models.Model): + """ + Represents the action of a user importing a modulestore-based course or legacy + library into a learning-core based learning package (today, that is always a content library). + """ + + user_task_status = models.OneToOneField( + 'user_tasks.UserTaskStatus', + on_delete=models.SET_NULL, + null=True, + blank=True, + related_name='import_event', + ) + user = models.ForeignKey(User, on_delete=models.CASCADE) + + # Note: For now, this will always be a course key. In the future, it may be a legacy library key. + source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True) + target_change = models.ForeignKey(to='oel_publishing.DraftChangeLog', on_delete=models.SET_NULL, null=True) + composition_level = models.CharField( + max_length=255, + choices=CompositionLevel.choices(), + help_text=_('The composition level of the target learning package'), + ) + override = models.BooleanField( + default=False, + help_text=_( + 'If true, the import will override any existing content in the target learning package.' + ), + ) + + class Meta: + verbose_name = _('Import from modulestore') + verbose_name_plural = _('Imports from modulestore') + + def __str__(self): + return f'{self.source_key} → {self.target_change}' + + def set_status(self, status: ImportStatus): + """ + Set import status. + """ + if status in ImportStatus.FAILED_STATUSES.value: + self.user_task_status.fail(status) + elif status == ImportStatus.CANCELED: + self.user_task_status.cancel() + else: + self.user_task_status.set_state(status) + self.user_task_status.save() + if status in [ImportStatus.IMPORTED, ImportStatus.CANCELED]: + self.clean_related_staged_content() + + def clean_related_staged_content(self) -> None: + """ + Clean related staged content. + """ + for staged_content_for_import in self.staged_content_for_import.all(): + staged_content_for_import.staged_content.delete() + + +class PublishableEntityMapping(TimeStampedModel): + """ + Represents a mapping between a source usage key and a target publishable entity. + """ + + source_usage_key = UsageKeyField( + max_length=255, + help_text=_('Original usage key/ID of the thing that has been imported.'), + ) + target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE) + + class Meta: + unique_together = ('source_usage_key', 'target_package') + + def __str__(self): + return f'{self.source_usage_key} → {self.target_entity}' + + +class PublishableEntityImport(TimeStampedModel): + """ + Represents a publishableentity version that has been imported into a learning package (e.g. content library) + + This is a many-to-many relationship between a container version and a course to library import. + """ + + import_event = models.ForeignKey(Import, on_delete=models.CASCADE) + resulting_mapping = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True) + resulting_change = models.OneToOneField( + to='oel_publishing.DraftChangeLogRecord', + # a changelog record can be pruned, which would set this to NULL, but not delete the + # entire import record + null=True, + on_delete=models.SET_NULL, + ) + + class Meta: + unique_together = ( + ('import_event', 'resulting_mapping'), + ) + + def __str__(self): + return f'{self.import_event} → {self.resulting_mapping}' + + +class StagedContentForImport(TimeStampedModel): + """ + Represents m2m relationship between an import and staged content created for that import. + """ + + import_event = models.ForeignKey( + Import, + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + staged_content = models.OneToOneField( + to='content_staging.StagedContent', + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + # Since StagedContent stores all the keys of the saved blocks, this field was added to optimize search. + source_usage_key = UsageKeyField( + max_length=255, + help_text=_( + 'The original Usage key of the highest-level component that was saved in StagedContent.' + ), + ) + + class Meta: + unique_together = ( + ('import_event', 'staged_content'), + ) + + def __str__(self): + return f'{self.import_event} → {self.staged_content}' diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py new file mode 100644 index 000000000000..34a377d5e384 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tasks.py @@ -0,0 +1,146 @@ +""" +Tasks for course to library import. +""" +from typing import Sequence + +from celery import shared_task +from celery.utils.log import get_task_logger +from django.db import transaction +from edx_django_utils.monitoring import set_code_owner_attribute_from_module +from opaque_keys.edx.keys import UsageKey + +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import LearningPackage +from openedx.core.djangoapps.content_staging import api as content_staging_api +from user_tasks.tasks import UserTask + +from .constants import IMPORT_FROM_MODULESTORE_STAGING_PURPOSE +from .data import ImportStatus +from .helpers import get_items_to_import, import_from_staged_content +from .models import Import, PublishableEntityImport, StagedContentForImport + +log = get_task_logger(__name__) + + +class ImportToLibraryTask(UserTask): + """ + Base class for import to library tasks. + """ + + @staticmethod + def calculate_total_steps(arguments_dict): + """ + Get number of in-progress steps in importing process, as shown in the UI. + + For reference, these are: + + 1. Staging content + 2. Importing staged content into library + """ + return 2 + + @classmethod + def generate_name(cls, arguments_dict): + """ + Create a name for this particular import task instance. + + Arguments: + arguments_dict (dict): The arguments given to the task function + + Returns: + str: The generated name + """ + library_id = arguments_dict.get('learning_package_id') + import_id = arguments_dict.get('import_pk') + return f'Import course to library (library_id={library_id}, import_id={import_id})' + + +@shared_task(base=ImportToLibraryTask, bind=True) +# Note: The decorator @set_code_owner_attribute cannot be used here because the UserTaskMixin +# does stack inspection and can't handle additional decorators. +def import_to_library_task( + self, + import_pk: int, + usage_key_strings: Sequence[str | UsageKey], + learning_package_id: int, + user_id: int, +) -> None: + """ + Import to library task. + + 1. Save course (for now) to staged content task by sections/chapters. + 2. Import staged content to library task. + """ + set_code_owner_attribute_from_module(__name__) + + # Step 1: Save course to staged content task by sections/chapters. + try: + import_event = Import.objects.get(pk=import_pk) + import_event.user_task_status = self.status + import_event.save(update_fields=['user_task_status']) + except Import.DoesNotExist: + log.info('Import event not found for pk %s', import_pk) + return + + import_event.set_status(ImportStatus.WAITNG_TO_STAGE) + import_event.clean_related_staged_content() + + import_event.set_status(ImportStatus.STAGING) + try: + with transaction.atomic(): + items_to_import = get_items_to_import(import_event) + for item in items_to_import: + staged_content = content_staging_api.stage_xblock_temporarily( + item, + user_id, + purpose=IMPORT_FROM_MODULESTORE_STAGING_PURPOSE, + ) + StagedContentForImport.objects.create( + staged_content=staged_content, + import_event=import_event, + source_usage_key=item.location + ) + + if items_to_import: + import_event.set_status(ImportStatus.STAGED) + else: + import_event.set_status(ImportStatus.STAGING_FAILED) + except Exception as exc: # pylint: disable=broad-except + import_event.set_status(ImportStatus.STAGING_FAILED) + raise exc + + # Step 2: Import staged content to library task. + self.status.increment_completed_steps() + + target_learning_package = LearningPackage.objects.get(id=learning_package_id) + + imported_publishable_versions = [] + import_event.set_status(ImportStatus.IMPORTING) + with authoring_api.bulk_draft_changes_for(learning_package_id=learning_package_id) as change_log: + try: + for usage_key_string in usage_key_strings: + staged_content_for_import = import_event.staged_content_for_import.get( + source_usage_key=usage_key_string + ) + publishable_versions = import_from_staged_content( + import_event, + usage_key_string, + target_learning_package, + staged_content_for_import.staged_content, + import_event.composition_level, + import_event.override, + ) + imported_publishable_versions.extend(publishable_versions) + except Exception as exc: # pylint: disable=broad-except + import_event.set_status(ImportStatus.IMPORTING_FAILED) + raise exc + else: + import_event.set_status(ImportStatus.IMPORTED) + for imported_component_version in imported_publishable_versions: + PublishableEntityImport.objects.create( + import_event=import_event, + resulting_mapping=imported_component_version.mapping, + resulting_change=change_log.records.get(entity=imported_component_version.mapping.target_entity), + ) + finally: + import_event.clean_related_staged_content() diff --git a/cms/djangoapps/import_from_modulestore/tests/factories.py b/cms/djangoapps/import_from_modulestore/tests/factories.py new file mode 100644 index 000000000000..e8423b3a84b5 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/factories.py @@ -0,0 +1,26 @@ +""" +Factories for Import model. +""" +import uuid + +import factory +from factory.django import DjangoModelFactory +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.models import Import + + +class ImportFactory(DjangoModelFactory): + """ + Factory for Import model. + """ + + class Meta: + model = Import + + @factory.lazy_attribute + def source_key(self): + return CourseKey.from_string(f'course-v1:edX+DemoX+{uuid.uuid4()}') + + user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py new file mode 100644 index 000000000000..72497200a7f9 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_api.py @@ -0,0 +1,98 @@ +""" +Test cases for import_from_modulestore.api module. +""" +from unittest.mock import patch + +import pytest +from opaque_keys.edx.keys import CourseKey +from organizations.models import Organization + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.api import import_to_library +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from .factories import ImportFactory + + +@pytest.mark.django_db +class TestCourseToLibraryImportAPI(ModuleStoreTestCase): + """ + Test cases for Import API. + """ + + def setUp(self): + super().setUp() + + _library_metadata = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.library = content_libraries_api.ContentLibrary.objects.get_by_key(_library_metadata.key) + + def test_import_staged_content_to_library(self): + """ + Test import_staged_content_to_library function with different override values. + """ + user = UserFactory() + source_key = "course-v1:edX+DemoX+Demo_Course" + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@chapter+block@123", + "block-v1:edX+DemoX+Demo_Course+type@chapter+block@456", + ] + override = False + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_to_library_task" + ) as import_to_library_task_mock: + import_to_library( + source_key=source_key, + usage_ids=usage_ids, + target_learning_package_id=self.library.learning_package.id, + user_id=user.id, + composition_level="component", + override=override, + ) + + import_event = Import.objects.get() + assert str(import_event.source_key) == source_key + assert import_event.composition_level == "component" + assert import_event.override == override + assert import_event.user_id == user.id + + import_to_library_task_mock.apply_async.assert_called_once_with( + kwargs={ + "import_pk": Import.objects.get().pk, + "usage_key_strings": usage_ids, + "learning_package_id": self.library.learning_package.id, + "user_id": user.id, + }, + ) + + def test_import_to_library_invalid_usage_key(self): + """ + Test import_staged_content_to_library function with not chapter usage keys. + """ + import_event = ImportFactory( + source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"), + ) + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@problem+block@123", + "block-v1:edX+DemoX+Demo_Course+type@vertical+block@456", + ] + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_to_library_task" + ) as import_to_library_task_mock: + with self.assertRaises(ValueError): + import_to_library( + source_key=import_event.source_key, + usage_ids=usage_ids, + target_learning_package_id=self.library.learning_package.id, + user_id=import_event.user.id, + composition_level="component", + override=False, + ) + import_to_library_task_mock.apply_async.assert_not_called() diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py new file mode 100644 index 000000000000..5e5aad1b1c84 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py @@ -0,0 +1,442 @@ +""" +Tests for the import_from_modulestore helper functions. +""" +import uuid + +import ddt +from organizations.models import Organization +from unittest import mock +from unittest.mock import patch + +from lxml import etree +from openedx_learning.api.authoring_models import LearningPackage +from user_tasks.models import UserTaskStatus + +from cms.djangoapps.import_from_modulestore import api, constants +from cms.djangoapps.import_from_modulestore.helpers import ImportClient +from cms.djangoapps.import_from_modulestore.models import StagedContentForImport +from cms.djangoapps.import_from_modulestore.tests.factories import ImportFactory +from common.djangoapps.student.tests.factories import UserFactory + +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory +from xmodule.modulestore.django import modulestore + + +@ddt.ddt +class TestImportClient(ModuleStoreTestCase): + """ + Functional tests for the ImportClient class. + """ + + def setUp(self): + super().setUp() + self.library = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.learning_package = LearningPackage.objects.get(id=self.library.learning_package_id) + self.user = UserFactory() + self.course = CourseFactory.create() + self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter') + self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential') + self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical') + self.problem = BlockFactory.create( + category='problem', + parent=self.vertical, + display_name='Problem', + data="""""", + ) + self.video = BlockFactory.create( + category='video', + parent=self.vertical, + display_name='Video', + data="""