diff --git a/.importlinter b/.importlinter index 0317245a6..6c278d0dd 100644 --- a/.importlinter +++ b/.importlinter @@ -35,6 +35,9 @@ layers= # The public authoring API is at the top–none of the apps should call to it. openedx_learning.api.authoring + # The "backup_restore" app handle the new export and import mechanism. + openedx_learning.apps.authoring.backup_restore + # The "components" app is responsible for storing versioned Components, # which is Open edX Studio terminology maps to things like individual # Problems, Videos, and blocks of HTML text. This is also the type we would @@ -50,9 +53,6 @@ layers= # Its only dependency should be the publishing app. openedx_learning.apps.authoring.collections - # The "backup_restore" app handle the new export and import mechanism. - openedx_learning.apps.authoring.backup_restore - # The lowest layer is "publishing", which holds the basic primitives needed # to create Learning Packages and manage the draft and publish states for # various types of content. diff --git a/openedx_learning/api/authoring.py b/openedx_learning/api/authoring.py index 5db14ceff..9082b33fb 100644 --- a/openedx_learning/api/authoring.py +++ b/openedx_learning/api/authoring.py @@ -9,6 +9,7 @@ """ # These wildcard imports are okay because these api modules declare __all__. # pylint: disable=wildcard-import +from ..apps.authoring.backup_restore.api import * from ..apps.authoring.collections.api import * from ..apps.authoring.components.api import * from ..apps.authoring.contents.api import * diff --git a/openedx_learning/apps/authoring/backup_restore/api.py b/openedx_learning/apps/authoring/backup_restore/api.py index b26fc207a..5dc66b98d 100644 --- a/openedx_learning/apps/authoring/backup_restore/api.py +++ b/openedx_learning/apps/authoring/backup_restore/api.py @@ -1,14 +1,9 @@ """ Backup Restore API """ -import zipfile - +from openedx_learning.apps.authoring.backup_restore.zipper import LearningPackageZipper from openedx_learning.apps.authoring.publishing.api import get_learning_package_by_key -from .toml import TOMLLearningPackageFile - -TOML_PACKAGE_NAME = "package.toml" - def create_zip_file(lp_key: str, path: str) -> None: """ @@ -17,9 +12,4 @@ def create_zip_file(lp_key: str, path: str) -> None: Can throw a NotFoundError at get_learning_package_by_key """ learning_package = get_learning_package_by_key(lp_key) - toml_file = TOMLLearningPackageFile(learning_package) - toml_file.create() - toml_content: str = toml_file.get() - with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: - # Add the TOML string as a file in the ZIP - zipf.writestr(TOML_PACKAGE_NAME, toml_content) + LearningPackageZipper(learning_package).create_zip(path) diff --git a/openedx_learning/apps/authoring/backup_restore/management/commands/lp_dump.py b/openedx_learning/apps/authoring/backup_restore/management/commands/lp_dump.py index 12acf2f04..f1b9ddfe9 100644 --- a/openedx_learning/apps/authoring/backup_restore/management/commands/lp_dump.py +++ b/openedx_learning/apps/authoring/backup_restore/management/commands/lp_dump.py @@ -33,7 +33,6 @@ def handle(self, *args, **options): self.stdout.write(self.style.SUCCESS(message)) except LearningPackage.DoesNotExist as exc: message = f"Learning package with key {lp_key} not found" - logger.exception(message) raise CommandError(message) from exc except Exception as e: message = f"Failed to export learning package '{lp_key}': {e}" diff --git a/openedx_learning/apps/authoring/backup_restore/toml.py b/openedx_learning/apps/authoring/backup_restore/toml.py index fd3854204..05ede1df3 100644 --- a/openedx_learning/apps/authoring/backup_restore/toml.py +++ b/openedx_learning/apps/authoring/backup_restore/toml.py @@ -1,72 +1,73 @@ """ -Utilities for backup and restore app +TOML serialization for learning packages and publishable entities. """ from datetime import datetime -from typing import Any, Dict -from tomlkit import comment, document, dumps, nl, table -from tomlkit.items import Table +import tomlkit from openedx_learning.apps.authoring.publishing.models.learning_package import LearningPackage +from openedx_learning.apps.authoring.publishing.models.publishable_entity import ( + PublishableEntityMixin, + PublishableEntityVersionMixin, +) -class TOMLLearningPackageFile(): - """ - Class to create a .toml representation of a LearningPackage instance. +def toml_learning_package(learning_package: LearningPackage) -> str: + """Create a TOML representation of the learning package.""" + doc = tomlkit.document() + doc.add(tomlkit.comment(f"Datetime of the export: {datetime.now()}")) + section = tomlkit.table() + section.add("title", learning_package.title) + section.add("key", learning_package.key) + section.add("description", learning_package.description) + section.add("created", learning_package.created) + section.add("updated", learning_package.updated) + doc.add("learning_package", section) + return tomlkit.dumps(doc) - This class builds a structured TOML document using `tomlkit` with metadata and fields - extracted from a `LearningPackage` object. The output can later be saved to a file or used elsewhere. - """ - def __init__(self, learning_package: LearningPackage): - self.doc = document() - self.learning_package = learning_package +def toml_publishable_entity(entity: PublishableEntityMixin) -> str: + """Create a TOML representation of a publishable entity.""" + doc = tomlkit.document() + entity_table = tomlkit.table() + entity_table.add("uuid", str(entity.uuid)) + entity_table.add("can_stand_alone", entity.can_stand_alone) - def _create_header(self) -> None: - """ - Adds a comment with the current datetime to indicate when the export occurred. - This helps with traceability and file versioning. - """ - self.doc.add(comment(f"Datetime of the export: {datetime.now()}")) - self.doc.add(nl()) + draft = tomlkit.table() + draft.add("version_num", entity.versioning.draft.version_num) + entity_table.add("draft", draft) - def _create_table(self, params: Dict[str, Any]) -> Table: - """ - Builds a TOML table section from a dictionary of key-value pairs. + published = tomlkit.table() + if entity.versioning.published: + published.add("version_num", entity.versioning.published.version_num) + else: + published.add(tomlkit.comment("unpublished: no published_version_num")) + entity_table.add("published", published) - Args: - params (Dict[str, Any]): A dictionary containing keys and values to include in the TOML table. + doc.add("entity", entity_table) + doc.add(tomlkit.nl()) + doc.add(tomlkit.comment("### Versions")) - Returns: - Table: A TOML table populated with the provided keys and values. - """ - section = table() - for key, value in params.items(): - section.add(key, value) - return section + for entity_version in entity.versioning.versions.all(): + version = tomlkit.aot() + version_table = toml_publishable_entity_version(entity_version) + version.append(version_table) + doc.add("version", version) - def create(self) -> None: - """ - Populates the TOML document with a header and a table containing - metadata from the LearningPackage instance. + return tomlkit.dumps(doc) - This method must be called before calling `get()`, otherwise the document will be empty. - """ - self._create_header() - section = self._create_table({ - "title": self.learning_package.title, - "key": self.learning_package.key, - "description": self.learning_package.description, - "created": self.learning_package.created, - "updated": self.learning_package.updated - }) - self.doc.add("learning_package", section) - def get(self) -> str: - """ - Returns: - str: The string representation of the generated TOML document. - Ensure `create()` has been called beforehand to get meaningful output. - """ - return dumps(self.doc) +def toml_publishable_entity_version(version: PublishableEntityVersionMixin) -> tomlkit.items.Table: + """Create a TOML representation of a publishable entity version.""" + version_table = tomlkit.table() + version_table.add("title", version.title) + version_table.add("uuid", str(version.uuid)) + version_table.add("version_num", version.version_num) + container_table = tomlkit.table() + container_table.add("children", []) + unit_table = tomlkit.table() + unit_table.add("graded", True) + container_table.add("unit", unit_table) + version_table.add("container", container_table) + return version_table # For use in AoT diff --git a/openedx_learning/apps/authoring/backup_restore/zipper.py b/openedx_learning/apps/authoring/backup_restore/zipper.py new file mode 100644 index 000000000..2c07401e5 --- /dev/null +++ b/openedx_learning/apps/authoring/backup_restore/zipper.py @@ -0,0 +1,53 @@ +""" +This module provides functionality to create a zip file containing the learning package data, +including a TOML representation of the learning package and its entities. +""" +import zipfile +from pathlib import Path + +from openedx_learning.apps.authoring.backup_restore.toml import toml_learning_package, toml_publishable_entity +from openedx_learning.apps.authoring.components import api as components_api +from openedx_learning.apps.authoring.publishing.models.learning_package import LearningPackage + +TOML_PACKAGE_NAME = "package.toml" + + +class LearningPackageZipper: + """ + A class to handle the zipping of learning content for backup and restore. + """ + + def __init__(self, learning_package: LearningPackage): + self.learning_package = learning_package + + def create_zip(self, path: str) -> None: + """ + Creates a zip file containing the learning package data. + Args: + path (str): The path where the zip file will be created. + Raises: + Exception: If the learning package cannot be found or if the zip creation fails. + """ + package_toml_content: str = toml_learning_package(self.learning_package) + + with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: + # Add the package.toml string + zipf.writestr(TOML_PACKAGE_NAME, package_toml_content) + + # Add the entities directory + entities_folder = Path("entities") + zip_info = zipfile.ZipInfo(str(entities_folder) + "/") # Ensure trailing slash + zipf.writestr(zip_info, "") # Add explicit empty directory entry + + # Add the collections directory + collections_folder = Path("collections") + collections_info = zipfile.ZipInfo(str(collections_folder) + "/") # Ensure trailing slash + zipf.writestr(collections_info, "") # Add explicit empty directory + + # Add each entity's TOML file + for entity in components_api.get_components(self.learning_package.pk): + # Create a TOML representation of the entity + entity_toml_content: str = toml_publishable_entity(entity) + entity_toml_filename = f"{entity.key}.toml" + entity_toml_path = entities_folder / entity_toml_filename + zipf.writestr(str(entity_toml_path), entity_toml_content) diff --git a/openedx_learning/apps/authoring/publishing/models/publishable_entity.py b/openedx_learning/apps/authoring/publishing/models/publishable_entity.py index 0a95e0a9e..d967a343b 100644 --- a/openedx_learning/apps/authoring/publishing/models/publishable_entity.py +++ b/openedx_learning/apps/authoring/publishing/models/publishable_entity.py @@ -294,6 +294,10 @@ def versioning(self): def uuid(self) -> str: return self.publishable_entity.uuid + @property + def can_stand_alone(self) -> bool: + return self.publishable_entity.can_stand_alone + @property def key(self) -> str: return self.publishable_entity.key diff --git a/test_settings.py b/test_settings.py index e1e4d79ab..a50efb17c 100644 --- a/test_settings.py +++ b/test_settings.py @@ -48,6 +48,7 @@ def root(*args): "openedx_learning.apps.authoring.sections.apps.SectionsConfig", "openedx_learning.apps.authoring.subsections.apps.SubsectionsConfig", "openedx_learning.apps.authoring.units.apps.UnitsConfig", + "openedx_learning.apps.authoring.backup_restore.apps.BackupRestoreConfig", ] AUTHENTICATION_BACKENDS = [ diff --git a/tests/openedx_learning/apps/authoring/backup_restore/__init__.py b/tests/openedx_learning/apps/authoring/backup_restore/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py new file mode 100644 index 000000000..2ffe3253c --- /dev/null +++ b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py @@ -0,0 +1,208 @@ +""" +Tests relating to dumping learning packages to disk +""" +import zipfile +from datetime import datetime, timezone +from io import StringIO +from pathlib import Path + +from django.contrib.auth import get_user_model +from django.core.management import CommandError, call_command +from django.db.models import QuerySet + +from openedx_learning.api import authoring as api +from openedx_learning.api.authoring_models import Component, LearningPackage +from openedx_learning.lib.test_utils import TestCase + +User = get_user_model() + + +class LpDumpCommandTestCase(TestCase): + """ + Test serving static assets (Content files, via Component lookup). + """ + + learning_package: LearningPackage + all_components: QuerySet[Component] + + @classmethod + def setUpTestData(cls): + """ + Initialize our content data + """ + + # Create a user for the test + cls.user = User.objects.create( + username="user", + email="user@example.com", + ) + + # Create a Learning Package for the test + cls.learning_package = api.create_learning_package( + key="ComponentTestCase-test-key", + title="Components Test Case Learning Package", + description="This is a test learning package for components.", + ) + cls.now = datetime(2024, 8, 5, tzinfo=timezone.utc) + + cls.xblock_v1_namespace = "xblock.v1" + + # Create component types + cls.html_type = api.get_or_create_component_type(cls.xblock_v1_namespace, "html") + cls.problem_type = api.get_or_create_component_type(cls.xblock_v1_namespace, "problem") + + # Make and publish one Component + cls.published_component, _ = api.create_component_and_version( + cls.learning_package.id, + cls.problem_type, + local_key="my_published_example", + title="My published problem", + created=cls.now, + created_by=cls.user.id, + ) + + # Create a Content entry for the published Component + api.publish_all_drafts( + cls.learning_package.id, + message="Publish from CollectionTestCase.setUpTestData", + published_at=cls.now, + ) + + # Create a Draft component, one in each learning package + cls.draft_component, _ = api.create_component_and_version( + cls.learning_package.id, + cls.html_type, + local_key="my_draft_example", + title="My draft html", + created=cls.now, + created_by=cls.user.id, + ) + + api.create_component_version( + cls.draft_component.pk, + version_num=cls.draft_component.versioning.draft.version_num + 1, + title="My draft html v2", + created=cls.now, + created_by=cls.user.id, + ) + + # components = self.learning_package.publishable_entities.all() + components = api.get_components(cls.learning_package) + cls.all_components = components + + def check_toml_file(self, zip_path: Path, zip_member_name: Path, content_to_check: list): + """ + Check that a specific entity TOML file in the zip matches the expected content. + """ + with zipfile.ZipFile(zip_path, "r") as zip_file: + with zip_file.open(str(zip_member_name)) as toml_file: + toml_content = toml_file.read().decode("utf-8") + for value in content_to_check: + self.assertIn(value, toml_content) + + def check_zip_file_structure(self, zip_path: Path): + """ + Check that the zip file has the expected structure. + """ + + with zipfile.ZipFile(zip_path, "r") as zip_file: + # Check that the zip file contains the expected files + expected_files = [ + "package.toml", + "entities/", + "collections/", + ] + + # Add expected entity files + for entity in self.all_components: + expected_files.append(f"entities/{entity.key}.toml") + + # Check that all expected files are present + for expected_file in expected_files: + self.assertIn(expected_file, zip_file.namelist()) + + def check_content_in_zip(self, zip_path: Path, expected_file_path: Path, zip_member_name: str): + """ + Compare a file inside the zip with an expected file. + """ + with zipfile.ZipFile(zip_path, "r") as zip_file: + with zip_file.open(zip_member_name) as content_file: + actual_content = content_file.read().decode("utf-8") + expected_content = expected_file_path.read_text() + self.assertEqual(actual_content, expected_content) + + def test_lp_dump_command(self): + lp_key = self.learning_package.key + file_name = f"{lp_key}.zip" + try: + out = StringIO() + + # Call the management command to dump the learning package + call_command("lp_dump", lp_key, file_name, stdout=out) + + # Check that the zip file was created + self.assertTrue(Path(file_name).exists()) + + # Check the structure of the zip file + self.check_zip_file_structure(Path(file_name)) + + zip_path = Path(file_name) + + # Check the content of the package.toml file + self.check_toml_file( + zip_path, + Path("package.toml"), + [ + '[learning_package]', + f'key = "{self.learning_package.key}"', + f'title = "{self.learning_package.title}"', + f'description = "{self.learning_package.description}"', + ] + ) + + # Check the content of the entity TOML files + for entity in self.all_components: + expected_content = [ + '[entity]', + f'uuid = "{entity.uuid}"', + 'can_stand_alone = true', + '[entity.draft]', + f'version_num = {entity.versioning.draft.version_num}', + '[entity.published]', + ] + if entity.versioning.published: + expected_content.append(f'version_num = {entity.versioning.published.version_num}') + else: + expected_content.append('# unpublished: no published_version_num') + + for entity_version in entity.versioning.versions.all(): + expected_content.append(f'title = "{entity_version.title}"') + expected_content.append(f'uuid = "{entity_version.uuid}"') + expected_content.append(f'version_num = {entity_version.version_num}') + expected_content.append('[version.container]') + expected_content.append('[version.container.unit]') + + self.check_toml_file( + zip_path, + Path(f"entities/{entity.key}.toml"), + expected_content + ) + + # Check the output message + message = f'{lp_key} written to {file_name}' + self.assertIn(message, out.getvalue()) + except Exception as e: # pylint: disable=broad-exception-caught + self.fail(f"lp_dump command failed with error: {e}") + finally: + # Clean up the created zip file + if Path(file_name).exists(): + Path(file_name).unlink(missing_ok=True) + + def test_dump_nonexistent_learning_package(self): + out = StringIO() + lp_key = "nonexistent_lp" + file_name = f"{lp_key}.zip" + with self.assertRaises(CommandError): + # Attempt to dump a learning package that does not exist + call_command("lp_dump", lp_key, file_name, stdout=out) + self.assertIn("Learning package 'nonexistent_lp' does not exist", out.getvalue())