From 958b903df8fd5767b3ccdee04a4cda2adeec4ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Thu, 17 Apr 2025 14:29:26 +0300 Subject: [PATCH 01/26] feat: create import client --- .../import_from_modulestore/helpers.py | 385 ++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 cms/djangoapps/import_from_modulestore/helpers.py diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py new file mode 100644 index 000000000000..b432ef20f5d3 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -0,0 +1,385 @@ +""" +Helper functions for importing course content into a library. +""" +from datetime import datetime, timezone +import logging +import mimetypes +import os +import secrets + +from django.db import transaction +from django.db.utils import IntegrityError +from lxml import etree + +from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.locator import CourseLocator, LibraryUsageLocatorV2 +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import ContainerVersion + +from openedx.core.djangoapps.content_libraries import api +from openedx.core.djangoapps.content_libraries.api.exceptions import IncompatibleTypesError +from openedx.core.djangoapps.content_staging import api as content_staging_api +from xmodule.modulestore.django import modulestore + +from .data import CompositionLevel +from .models import Import, PublishableEntityMapping, PublishableEntityImport + + +log = logging.getLogger(__name__) +parser = etree.XMLParser(strip_cdata=False) + + +class ImportClient: + """ + Client for importing course content into a library. + + This class handles the import of course content from staged content into a + content library, creating the appropriate container hierarchy based on the + specified composition level. + """ + + CONTAINER_CREATORS_MAP = { + 'chapter': authoring_api.create_unit_and_version, # TODO: replace with create_module_and_version + 'sequential': authoring_api.create_unit_and_version, # TODO: replace with create_section_and_version + 'vertical': authoring_api.create_unit_and_version, + } + + CONTAINER_OVERRIDERS_MAP = { + 'chapter': authoring_api.create_next_unit_version, # TODO: replace with create_next_module_version + 'sequential': authoring_api.create_next_unit_version, # TODO: replace with create_next_section_version + 'vertical': authoring_api.create_next_unit_version, + } + + def __init__( + self, + import_event: Import, + block_usage_key_to_import: str, + staged_content: 'StagedContent', + composition_level: str, + override: bool = False, + ): + self.import_event = import_event + self.block_usage_key_to_import = block_usage_key_to_import + self.staged_content = staged_content + self.composition_level = composition_level + self.override = override + + self.user_id = import_event.user_id + self.content_library = import_event.target.contentlibrary + self.library_key = self.content_library.library_key + self.parser = etree.XMLParser(strip_cdata=False) + + def import_from_staged_content(self): + """ + Import staged content into a library. + """ + node = etree.fromstring(self.staged_content.olx, parser=parser) + usage_key = UsageKey.from_string(self.block_usage_key_to_import) + block_to_import = get_block_to_import(node, usage_key) + if block_to_import is None: + return + + self._process_import(self.block_usage_key_to_import, block_to_import) + + def _process_import(self, usage_id, block_to_import): + """ + Process import of a block from staged content into a library. + + Imports a block and its children into the library based on the + composition level. It handles both simple and complicated blocks, creating + the necessary container hierarchy. + """ + usage_key = UsageKey.from_string(usage_id) + result = [] + + if block_to_import.tag not in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_simple_block(block_to_import, usage_key) + + for child in block_to_import.getchildren(): + child_usage_key_string = get_usage_key_string_from_staged_content( + self.staged_content, child.get('url_name') + ) + if not child_usage_key_string: + continue + result.extend(self._import_child_block(child, child_usage_key_string)) + + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return [component for component in result if not isinstance(component, ContainerVersion)] + return result + + def _import_simple_block(self, block_to_import, usage_key) -> list: + """ + Import a simple block into the library. + + Creates a block in the library from the staged content block. + It returns a list containing the created component version. + """ + component_version = self._create_block_in_library(block_to_import, usage_key) + return [component_version] if component_version else [] + + def _import_child_block(self, child, child_usage_id): + """ + Import a child block into the library. + + Determines whether the child block is simple or complicated and + delegates the import process to the appropriate helper method. + """ + child_usage_key = UsageKey.from_string(child_usage_id) + if child.tag in CompositionLevel.COMPLICATED_LEVELS.value: + return self._import_complicated_child(child, child_usage_id) + else: + return self._import_simple_block(child, child_usage_key) + + def _import_complicated_child(self, child, child_usage_id): + """ + Import a complicated child block into the library. + + Handles the import of complicated child blocks, including creating + containers and updating components. + Returns a list containing the created container version. + """ + if self.composition_level in CompositionLevel.FLAT_LEVELS.value: + return self._process_import(child_usage_id, child) + + container_version = self.get_or_create_container( + child.tag, + child.get('url_name'), + child.get('display_name', child.tag) + ) + child_component_versions = self._process_import(child_usage_id, child) + self._update_container_components(container_version, child_component_versions) + return [container_version] + + def get_or_create_container(self, container_type, key, display_name): + """ + Create a container of the specified type. + + Creates a container (e.g., chapter, sequential, vertical) in the + content library. + """ + container_creator_func = self.CONTAINER_CREATORS_MAP.get(container_type) + container_override_func = self.CONTAINER_OVERRIDERS_MAP.get(container_type) + if not all((container_creator_func, container_override_func)): + raise ValueError(f"Unknown container type: {container_type}") + + container_version = self.content_library.learning_package.publishable_entities.filter(key=key).first() + if container_version and self.override: + container_version = container_override_func( + container_version.container, + title=display_name or f"New {container_type}", + components=[], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + elif not container_version: + _, container_version = container_creator_func( + self.import_event.target_id, + key=key or secrets.token_hex(16), + title=display_name or f"New {container_type}", + components=[], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + ) + + return container_version + + def _update_container_components(self, container_version, component_versions): + """ + Update components of a container. + """ + return authoring_api.create_next_container_version( + container_pk=container_version.container.pk, + title=container_version.title, + publishable_entities_pks=[ + cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions + ], + entity_version_pks=[cv.pk for cv in component_versions], + created=datetime.now(tz=timezone.utc), + created_by=self.import_event.user_id, + container_version_cls=container_version.__class__, + ) + + def _create_block_in_library(self, block_to_import, usage_key): + """ + Create a block in a library from a staged content block. + """ + now = datetime.now(tz=timezone.utc) + staged_content_files = content_staging_api.get_staged_content_static_files(self.staged_content.id) + + with transaction.atomic(): + component_type = authoring_api.get_or_create_component_type("xblock.v1", usage_key.block_type) + does_component_exist = authoring_api.get_components( + self.import_event.target_id + ).filter(local_key=usage_key.block_id).exists() + + if does_component_exist: + if not self.override: + log.info(f"Component {usage_key.block_id} already exists in library {self.library_key}, skipping.") + return + else: + component_version = self._handle_component_override(usage_key, etree.tostring(block_to_import)) + else: + try: + _, library_usage_key = api.validate_can_add_block_to_library( + self.library_key, + block_to_import.tag, + usage_key.block_id, + ) + except api.IncompatibleTypesError as e: + log.error(f"Error validating block {usage_key} for library {self.library_key}: {e}") + return + + authoring_api.create_component( + self.import_event.target_id, + component_type=component_type, + local_key=usage_key.block_id, + created=now, + created_by=self.import_event.user_id, + ) + component_version = api.set_library_block_olx(library_usage_key, etree.tostring(block_to_import)) + + self._process_staged_content_files( + component_version, + staged_content_files, + usage_key, + block_to_import, + now, + ) + _create_publishable_entity_import(self.import_event, usage_key, component_version) + + return component_version + + def _handle_component_override(self, usage_key, new_content): + """ + Create new ComponentVersion for overridden component. + """ + component_version = None + component = self.import_event.target.component_set.filter(local_key=usage_key.block_id).first() + + if component: + library_usage_key = LibraryUsageLocatorV2( # type: ignore[abstract] + lib_key=self.library_key, + block_type=component.component_type.name, + usage_id=component.local_key, + ) + component_version = api.set_library_block_olx(library_usage_key, new_content) + + return component_version + + def _process_staged_content_files( + self, + component_version, + staged_content_files, + usage_key, + block_to_import, + created_at, + ): + """ + Process staged content files for a component. + + Processes the staged content files for a component, creating the + necessary file content and associating it with the component version. + """ + block_olx = etree.tostring(block_to_import).decode('utf-8') + + for staged_content_file_data in staged_content_files: + original_filename = staged_content_file_data.filename + file_basename = os.path.basename(original_filename) + file_basename_no_ext, _ = os.path.splitext(file_basename) + + # Skip files not referenced in the block + if file_basename not in block_olx and file_basename_no_ext not in block_olx: + log.info(f"Skipping file {original_filename} as it is not referenced in block {usage_key}") + continue + + file_data = content_staging_api.get_staged_content_static_file_data( + self.staged_content.id, + original_filename, + ) + if not file_data: + log.error( + f"Staged content {self.staged_content.id} included referenced " + f"file {original_filename}, but no file data was found." + ) + continue + + filename = f"static/{file_basename}" + media_type_str, _ = mimetypes.guess_type(filename) + if not media_type_str: + media_type_str = "application/octet-stream" + + media_type = authoring_api.get_or_create_media_type(media_type_str) + content = authoring_api.get_or_create_file_content( + self.import_event.target_id, + media_type.id, + data=file_data, + created=created_at, + ) + + try: + authoring_api.create_component_version_content(component_version.pk, content.id, key=filename) + except IntegrityError: + pass # Content already exists + + +def _create_publishable_entity_import(import_event, usage_key, component_version) -> PublishableEntityImport: + """ + Creates relations between the imported component and source usage key and import event. + """ + publishable_entity_mapping, _ = _get_or_create_publishable_entity_mapping( + usage_key, + component_version.component + ) + return PublishableEntityImport.objects.create( + import_event=import_event, + result=publishable_entity_mapping, + resulting_draft=component_version.publishable_entity_version, + ) + + +def _get_or_create_publishable_entity_mapping(usage_key, component) -> tuple[PublishableEntityMapping, bool]: + """ + Creates a mapping between the source usage key and the target publishable entity. + """ + return PublishableEntityMapping.objects.get_or_create( + source_usage_key=usage_key, + target_entity=component.publishable_entity, + target_package=component.learning_package + ) + + +def get_usage_key_string_from_staged_content(staged_content, block_id): + """ + Get the usage ID from a staged content by block ID. + """ + return next((block_usage_id for block_usage_id in staged_content.tags if block_usage_id.endswith(block_id)), None) + + +def get_block_to_import(node, usage_key): + """ + Get the block to import from a node. + """ + + if node.get('url_name') == usage_key.block_id: + return node + + for child in node.getchildren(): + found = get_block_to_import(child, usage_key) + if found is not None: + return found + + +def get_items_to_import(import_event): + """ + Collect items to import from a course. + """ + items_to_import = [] + if isinstance(import_event.source_key, CourseLocator): + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "chapter"}) or [] + ) + items_to_import.extend( + modulestore().get_items(import_event.source_key, qualifiers={"category": "static_tab"}) or [] + ) + + return items_to_import From 4213966dfe9d12bbcba686e5bc9078ade60d8c5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Thu, 17 Apr 2025 14:47:10 +0300 Subject: [PATCH 02/26] feat: create celery tasks for import functionality --- .../import_from_modulestore/constants.py | 5 ++ .../import_from_modulestore/tasks.py | 89 +++++++++++++++++++ .../import_from_modulestore/validators.py | 31 +++++++ .../core/djangoapps/content_staging/api.py | 24 ++--- 4 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 cms/djangoapps/import_from_modulestore/constants.py create mode 100644 cms/djangoapps/import_from_modulestore/tasks.py create mode 100644 cms/djangoapps/import_from_modulestore/validators.py diff --git a/cms/djangoapps/import_from_modulestore/constants.py b/cms/djangoapps/import_from_modulestore/constants.py new file mode 100644 index 000000000000..099acfbc2b7b --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/constants.py @@ -0,0 +1,5 @@ +""" +Constants for import_from_modulestore app +""" + +IMPORT_FROM_MODULESTORE_PURPOSE = "import_from_modulestore" diff --git a/cms/djangoapps/import_from_modulestore/tasks.py b/cms/djangoapps/import_from_modulestore/tasks.py new file mode 100644 index 000000000000..643041c931e4 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tasks.py @@ -0,0 +1,89 @@ +""" +Tasks for course to library import. +""" + +from celery import shared_task +from celery.utils.log import get_task_logger +from django.db import transaction +from edx_django_utils.monitoring import set_code_owner_attribute + +from openedx.core.djangoapps.content_staging import api as content_staging_api + +from .constants import IMPORT_FROM_MODULESTORE_PURPOSE +from .data import ImportStatus +from .helpers import get_items_to_import, ImportClient +from .models import Import, StagedContentForImport +from .validators import validate_composition_level + +log = get_task_logger(__name__) + + +@shared_task +@set_code_owner_attribute +def save_legacy_content_to_staged_content_task(import_uuid: str) -> None: + """ + Save courses to staged content task by sections/chapters. + """ + try: + import_event = Import.objects.get(uuid=import_uuid) + except Import.DoesNotExist: + log.info('Import event not found for UUID %s', import_uuid) + return + + import_event.clean_related_staged_content() + try: + with transaction.atomic(): + items_to_import = get_items_to_import(import_event) + for item in items_to_import: + staged_content = content_staging_api.stage_xblock_temporarily( + item, + import_event.user.id, + purpose=IMPORT_FROM_MODULESTORE_PURPOSE, + ) + StagedContentForImport.objects.create( + staged_content=staged_content, + import_event=import_event, + source_usage_key=item.location + ) + + if items_to_import: + import_event.set_status(ImportStatus.READY) + else: + import_event.set_status(ImportStatus.ERROR) + except Exception as exc: # pylint: disable=broad-except + import_event.set_status(ImportStatus.ERROR) + raise exc + + +@shared_task +@set_code_owner_attribute +def import_course_staged_content_to_library_task( + usage_keys_string: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library task. + """ + validate_composition_level(composition_level) + try: + import_event = Import.objects.get(uuid=import_uuid, status=ImportStatus.READY, user_id=user_id) + except Import.DoesNotExist: + log.info('Ready import from modulestore not found') + return + + with transaction.atomic(): + for usage_key_string in usage_keys_string: + if staged_content_item := import_event.get_staged_content_by_source_usage_key(usage_key_string): # FIXME + import_client = ImportClient( + import_event, + usage_key_string, + staged_content_item, + composition_level, + override, + ) + import_client.import_from_staged_content() + + import_event.set_status(ImportStatus.IMPORTED) diff --git a/cms/djangoapps/import_from_modulestore/validators.py b/cms/djangoapps/import_from_modulestore/validators.py new file mode 100644 index 000000000000..c7c7fb79e65d --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/validators.py @@ -0,0 +1,31 @@ +""" +Validators for the import_from_modulestore app. +""" +from django.utils.translation import gettext_lazy as _ + +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import UsageKey + +from .data import CompositionLevel + + +def validate_usage_keys_to_import(usage_keys: list[str | UsageKey]): + """ + Validate the usage keys to import. + + Currently, supports importing from the modulestore only by chapters. + """ + for usage_key in usage_keys: + if isinstance(usage_key, str): + try: + usage_key = UsageKey.from_string(usage_key) + except InvalidKeyError: + raise ValueError(_(f'Invalid usage key: {usage_key}')) + if usage_key.block_type != 'chapter': + raise InvalidKeyError(_(f'Importing from modulestore only supports chapters, not {usage_key.block_type}')) + +def validate_composition_level(composition_level): + if composition_level not in CompositionLevel.values(): + raise ValueError( + _('Invalid composition level: {composition_level}').format(composition_level=composition_level) + ) diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index 7f8167d9aadd..7baae10baed4 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -53,17 +53,19 @@ def _save_xblock_to_staged_content( expired_ids = [] with transaction.atomic(): - # Mark all of the user's existing StagedContent rows as EXPIRED - to_expire = _StagedContent.objects.filter( - user_id=user_id, - purpose=purpose, - ).exclude( - status=StagedContentStatus.EXPIRED, - ) - for sc in to_expire: - expired_ids.append(sc.id) - sc.status = StagedContentStatus.EXPIRED - sc.save() + if purpose == CLIPBOARD_PURPOSE: + # Mark all of the user's existing StagedContent rows as EXPIRED + to_expire = _StagedContent.objects.filter( + user_id=user_id, + purpose=purpose, + ).exclude( + status=StagedContentStatus.EXPIRED, + ) + for sc in to_expire: + expired_ids.append(sc.id) + sc.status = StagedContentStatus.EXPIRED + sc.save() + # Insert a new StagedContent row for this staged_content = _StagedContent.objects.create( user_id=user_id, From c03dcd8082439df6a4696f8de4d301fb0fd3199e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Thu, 17 Apr 2025 14:48:23 +0300 Subject: [PATCH 03/26] feat: add python APIs to create and process import --- cms/djangoapps/import_from_modulestore/api.py | 43 +++++++++++++++++++ .../import_from_modulestore/signals.py | 26 +++++++++++ 2 files changed, 69 insertions(+) create mode 100644 cms/djangoapps/import_from_modulestore/api.py create mode 100644 cms/djangoapps/import_from_modulestore/signals.py diff --git a/cms/djangoapps/import_from_modulestore/api.py b/cms/djangoapps/import_from_modulestore/api.py new file mode 100644 index 000000000000..a6adee7ccb35 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/api.py @@ -0,0 +1,43 @@ +""" +API for course to library import. +""" +from opaque_keys.edx.keys import LearningContextKey + +from .models import Import as _Import +from .tasks import import_course_staged_content_to_library_task, save_legacy_content_to_staged_content_task +from .validators import validate_usage_keys_to_import + + +def create_import(source_key: LearningContextKey, user_id: int, learning_package_id: int) -> _Import: + """ + Create a new import event to import a course to a library and save course to staged content. + """ + import_from_modulestore = _Import.objects.create( + source_key=source_key, + target_id=learning_package_id, + user_id=user_id, + ) + save_legacy_content_to_staged_content_task.delay_on_commit(import_from_modulestore.uuid) + return import_from_modulestore + + +def import_course_staged_content_to_library( + usage_ids: list[str], + import_uuid: str, + user_id: int, + composition_level: str, + override: bool +) -> None: + """ + Import staged content to a library from staged content. + """ + validate_usage_keys_to_import(usage_ids) + import_course_staged_content_to_library_task.apply_async( + kwargs={ + 'usage_keys_string': usage_ids, + 'import_uuid': import_uuid, + 'user_id': user_id, + 'composition_level': composition_level, + 'override': override, + }, + ) diff --git a/cms/djangoapps/import_from_modulestore/signals.py b/cms/djangoapps/import_from_modulestore/signals.py new file mode 100644 index 000000000000..958c7e365f67 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/signals.py @@ -0,0 +1,26 @@ +""" +Signals for Import. +""" +from django.dispatch import receiver +from django.db.models.signals import post_save + +from .data import ImportStatus +from .models import Import + + +@receiver(post_save, sender=Import) +def cancel_incomplete_imports(sender, instance, created, **kwargs): + """ + Cancel any incomplete imports that have the same target as the current import. + + When a new import is created, we want to cancel any other incomplete user imports that have the same target. + """ + if created: + incomplete_user_imports_with_same_target = Import.objects.filter( + user=instance.user, + target=instance.target, + source_key=instance.source_key, + staged_content_for_import__isnull=False + ).exclude(uuid=instance.uuid) + for incomplete_import in incomplete_user_imports_with_same_target: + incomplete_import.set_status(ImportStatus.CANCELED) From a3ea142da1fa57512f495605aa0a970d73c3a889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Thu, 17 Apr 2025 16:22:49 +0300 Subject: [PATCH 04/26] style: fix pylint issues --- cms/djangoapps/import_from_modulestore/helpers.py | 13 ++++++++----- .../import_from_modulestore/validators.py | 14 +++----------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cms/djangoapps/import_from_modulestore/helpers.py b/cms/djangoapps/import_from_modulestore/helpers.py index b432ef20f5d3..f5e749c1df1c 100644 --- a/cms/djangoapps/import_from_modulestore/helpers.py +++ b/cms/djangoapps/import_from_modulestore/helpers.py @@ -17,7 +17,6 @@ from openedx_learning.api.authoring_models import ContainerVersion from openedx.core.djangoapps.content_libraries import api -from openedx.core.djangoapps.content_libraries.api.exceptions import IncompatibleTypesError from openedx.core.djangoapps.content_staging import api as content_staging_api from xmodule.modulestore.django import modulestore @@ -187,13 +186,17 @@ def _update_container_components(self, container_version, component_versions): """ Update components of a container. """ + entity_rows = [ + authoring_api.ContainerEntityRow( + entity_pk=cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk, + version_pk=cv.pk, + ) + for cv in component_versions + ] return authoring_api.create_next_container_version( container_pk=container_version.container.pk, title=container_version.title, - publishable_entities_pks=[ - cv.container.pk if isinstance(cv, ContainerVersion) else cv.component.pk for cv in component_versions - ], - entity_version_pks=[cv.pk for cv in component_versions], + entity_rows=entity_rows, created=datetime.now(tz=timezone.utc), created_by=self.import_event.user_id, container_version_cls=container_version.__class__, diff --git a/cms/djangoapps/import_from_modulestore/validators.py b/cms/djangoapps/import_from_modulestore/validators.py index c7c7fb79e65d..e7d4562c248b 100644 --- a/cms/djangoapps/import_from_modulestore/validators.py +++ b/cms/djangoapps/import_from_modulestore/validators.py @@ -1,9 +1,6 @@ """ Validators for the import_from_modulestore app. """ -from django.utils.translation import gettext_lazy as _ - -from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey from .data import CompositionLevel @@ -17,15 +14,10 @@ def validate_usage_keys_to_import(usage_keys: list[str | UsageKey]): """ for usage_key in usage_keys: if isinstance(usage_key, str): - try: - usage_key = UsageKey.from_string(usage_key) - except InvalidKeyError: - raise ValueError(_(f'Invalid usage key: {usage_key}')) + usage_key = UsageKey.from_string(usage_key) if usage_key.block_type != 'chapter': - raise InvalidKeyError(_(f'Importing from modulestore only supports chapters, not {usage_key.block_type}')) + raise ValueError(f'Importing from modulestore only supports chapters, not {usage_key.block_type}') def validate_composition_level(composition_level): if composition_level not in CompositionLevel.values(): - raise ValueError( - _('Invalid composition level: {composition_level}').format(composition_level=composition_level) - ) + raise ValueError(f'Invalid composition level: {composition_level}') From 9acd085be70baeb832eb29aa7fe49e31e2dcab9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=86=D0=B2=D0=B0=D0=BD=20=D0=9D=D1=94=D0=B4=D1=94=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=96=D1=86=D0=B5=D0=B2?= Date: Thu, 17 Apr 2025 17:01:38 +0300 Subject: [PATCH 05/26] test: add tests --- .../import_from_modulestore/tests/__init__.py | 0 .../tests/factories.py | 30 ++ .../import_from_modulestore/tests/test_api.py | 102 +++++ .../tests/test_helpers.py | 370 ++++++++++++++++++ .../tests/test_tasks.py | 165 ++++++++ .../tests/test_validators.py | 31 ++ .../import_from_modulestore/validators.py | 1 + .../content_libraries/tests/factories.py | 39 ++ setup.cfg | 1 + 9 files changed, 739 insertions(+) create mode 100644 cms/djangoapps/import_from_modulestore/tests/__init__.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/factories.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_api.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_helpers.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_tasks.py create mode 100644 cms/djangoapps/import_from_modulestore/tests/test_validators.py create mode 100644 openedx/core/djangoapps/content_libraries/tests/factories.py diff --git a/cms/djangoapps/import_from_modulestore/tests/__init__.py b/cms/djangoapps/import_from_modulestore/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/cms/djangoapps/import_from_modulestore/tests/factories.py b/cms/djangoapps/import_from_modulestore/tests/factories.py new file mode 100644 index 000000000000..2207a799ef33 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/factories.py @@ -0,0 +1,30 @@ +""" +Factories for Import model. +""" + +import uuid + +import factory +from factory.django import DjangoModelFactory +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories + + +class ImportFactory(DjangoModelFactory): + """ + Factory for Import model. + """ + + class Meta: + model = Import + + @factory.lazy_attribute + def source_key(self): + return CourseKey.from_string(f'course-v1:edX+DemoX+{self.uuid}') + + target = factory.SubFactory(factories.LearningPackageFactory) + uuid = factory.LazyFunction(lambda: str(uuid.uuid4())) + user = factory.SubFactory(UserFactory) diff --git a/cms/djangoapps/import_from_modulestore/tests/test_api.py b/cms/djangoapps/import_from_modulestore/tests/test_api.py new file mode 100644 index 000000000000..5627b8b89e4f --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_api.py @@ -0,0 +1,102 @@ +""" +Test cases for import_from_modulestore.api module. +""" +from unittest.mock import patch + +import pytest +from opaque_keys.edx.keys import CourseKey + +from common.djangoapps.student.tests.factories import UserFactory +from cms.djangoapps.import_from_modulestore.api import create_import, import_course_staged_content_to_library +from cms.djangoapps.import_from_modulestore.data import ImportStatus +from cms.djangoapps.import_from_modulestore.models import Import +from openedx.core.djangoapps.content_libraries.tests import factories +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from .factories import ImportFactory + + +@pytest.mark.django_db +class TestCourseToLibraryImportAPI(ModuleStoreTestCase): + """ + Test cases for Import API. + """ + + def setUp(self): + super().setUp() + + self.library = factories.ContentLibraryFactory() + + def test_create_import(self): + """ + Test create_import function. + """ + course_id = "course-v1:edX+DemoX+Demo_Course" + user = UserFactory() + create_import(course_id, user.id, self.library.learning_package_id) + + import_event = Import.objects.get() + assert import_event.source_key == CourseKey.from_string(course_id) + assert import_event.target == self.library.learning_package + assert import_event.user_id == user.id + assert import_event.status == ImportStatus.PENDING + + def test_import_course_staged_content_to_library(self): + """ + Test import_course_staged_content_to_library function with different override values. + """ + import_event = ImportFactory( + target=self.library.learning_package, + source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"), + ) + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@chapter+block@123", + "block-v1:edX+DemoX+Demo_Course+type@chapter+block@456", + ] + override = False + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_course_staged_content_to_library_task" + ) as import_course_staged_content_to_library_task_mock: + import_course_staged_content_to_library( + usage_ids, + import_event.uuid, + import_event.user.id, + "xblock", + override + ) + + import_course_staged_content_to_library_task_mock.apply_async.assert_called_once_with( + kwargs={ + "usage_keys_string": usage_ids, + "import_uuid": import_event.uuid, + "user_id": import_event.user.id, + "composition_level": "xblock", + "override": override, + }, + ) + + def test_import_course_staged_content_to_library_invalid_usage_key(self): + """ + Test import_course_staged_content_to_library function with not chapter usage keys. + """ + import_event = ImportFactory( + target=self.library.learning_package, + source_key=CourseKey.from_string("course-v1:edX+DemoX+Demo_Course"), + ) + usage_ids = [ + "block-v1:edX+DemoX+Demo_Course+type@problem+block@123", + "block-v1:edX+DemoX+Demo_Course+type@vertical+block@456", + ] + + with patch( + "cms.djangoapps.import_from_modulestore.api.import_course_staged_content_to_library_task" + ) as import_course_staged_content_to_library_task_mock: + with self.assertRaises(ValueError): + import_course_staged_content_to_library( + usage_ids, + import_event.uuid, + import_event.user.id, + "xblock", + False + ) + import_course_staged_content_to_library_task_mock.apply_async.assert_not_called() diff --git a/cms/djangoapps/import_from_modulestore/tests/test_helpers.py b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py new file mode 100644 index 000000000000..9c0b56d45bbd --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/tests/test_helpers.py @@ -0,0 +1,370 @@ +""" +Tests for the import_from_modulestore helper functions. +""" +import ddt +from organizations.models import Organization +from unittest import mock +from unittest.mock import patch + +from lxml import etree +from openedx_learning.api.authoring_models import LearningPackage + +from cms.djangoapps.import_from_modulestore import api +from cms.djangoapps.import_from_modulestore.helpers import ImportClient +from common.djangoapps.student.tests.factories import UserFactory + +from openedx.core.djangoapps.content_libraries import api as content_libraries_api +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory + + +@ddt.ddt +class TestImportClient(ModuleStoreTestCase): + """ + Functional tests for the ImportClient class. + """ + + def setUp(self): + super().setUp() + self.library = content_libraries_api.create_library( + org=Organization.objects.create(name='Organization 1', short_name='org1'), + slug='lib_1', + title='Library Org 1', + description='This is a library from Org 1', + ) + self.user = UserFactory() + self.course = CourseFactory.create() + self.chapter = BlockFactory.create(category='chapter', parent=self.course, display_name='Chapter') + self.sequential = BlockFactory.create(category='sequential', parent=self.chapter, display_name='Sequential') + self.vertical = BlockFactory.create(category='vertical', parent=self.sequential, display_name='Vertical') + self.problem = BlockFactory.create( + category='problem', + parent=self.vertical, + display_name='Problem', + data="""""", + ) + self.video = BlockFactory.create( + category='video', + parent=self.vertical, + display_name='Video', + data="""