From 5359b9901c4b76b51ab1e66642e3a954cf5fd15e Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Wed, 6 Aug 2025 16:46:28 -0600 Subject: [PATCH 1/2] feat: serialize components using namespaced directory structure Implements new directory layout for component serialization based on: - Component namespace - Component type - Component identifier Each component is now stored in its respective folder structure with its TOML file, and optionally includes a `component_versions//` subdirectory if version data is available --- .../apps/authoring/backup_restore/zipper.py | 53 +++++++++++++++++++ .../authoring/backup_restore/test_backup.py | 2 + 2 files changed, 55 insertions(+) diff --git a/openedx_learning/apps/authoring/backup_restore/zipper.py b/openedx_learning/apps/authoring/backup_restore/zipper.py index 8ea73c71e..8175779a0 100644 --- a/openedx_learning/apps/authoring/backup_restore/zipper.py +++ b/openedx_learning/apps/authoring/backup_restore/zipper.py @@ -20,6 +20,15 @@ class LearningPackageZipper: def __init__(self, learning_package: LearningPackage): self.learning_package = learning_package + def create_folder(self, folder_name: Path, zip_file: zipfile.ZipFile) -> None: + """ + Create a folder for the zip file structure. + Args: + folder_name (Path): The path of the folder to create. + """ + zip_info = zipfile.ZipInfo(str(folder_name) + "/") + zip_file.writestr(zip_info, "") # Add explicit empty directory entry + def create_zip(self, path: str) -> None: """ Creates a zip file containing the learning package data. @@ -29,6 +38,7 @@ def create_zip(self, path: str) -> None: Exception: If the learning package cannot be found or if the zip creation fails. """ package_toml_content: str = toml_learning_package(self.learning_package) + folders_already_created = set() with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as zipf: # Add the package.toml string @@ -51,3 +61,46 @@ def create_zip(self, path: str) -> None: entity_toml_filename = f"{entity.key}.toml" entity_toml_path = entities_folder / entity_toml_filename zipf.writestr(str(entity_toml_path), entity_toml_content) + + # Create component directories if they exist + if hasattr(entity, 'component'): + component_namespace_folder = entities_folder / entity.component.component_type.namespace + # Example of component namespace is: "xblock.v1" + if component_namespace_folder not in folders_already_created: + self.create_folder(component_namespace_folder, zipf) + folders_already_created.add(component_namespace_folder) + + component_type_folder = component_namespace_folder / entity.component.component_type.name + # Example of component name is: "html" + if component_type_folder not in folders_already_created: + self.create_folder(component_type_folder, zipf) + folders_already_created.add(component_type_folder) + + component_name_folder = component_type_folder / entity.component.local_key # entity.key + # Example of component name is: "i-dont-like-the-sidebar-aa1645ade4a7" + if component_name_folder not in folders_already_created: + self.create_folder(component_name_folder, zipf) + folders_already_created.add(component_name_folder) + + # Add the entity TOML file inside the component folder as well + component_entity_toml_path = component_type_folder / f"{entity.component.local_key}.toml" + zipf.writestr(str(component_entity_toml_path), entity_toml_content) + + # Add component version folder + component_version_folder = component_name_folder / "component_versions" + if component_version_folder not in folders_already_created: + self.create_folder(component_version_folder, zipf) + folders_already_created.add(component_version_folder) + + for entity_version in entity.component.versions.all(): + component_number_version_folder = component_version_folder / f"v{entity_version.version_num}" + # Create a folder for each version of the component. Example: "v1", "v2", etc. + if component_number_version_folder not in folders_already_created: + self.create_folder(component_number_version_folder, zipf) + folders_already_created.add(component_number_version_folder) + + # Add the static folder inside the component version folder + static_folder = component_number_version_folder / "static" + if static_folder not in folders_already_created: + self.create_folder(static_folder, zipf) + folders_already_created.add(static_folder) diff --git a/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py index 2ffe3253c..0b7e9ec1d 100644 --- a/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py +++ b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py @@ -111,6 +111,8 @@ def check_zip_file_structure(self, zip_path: Path): "package.toml", "entities/", "collections/", + "entities/xblock.v1/", + "entities/xblock.v1/html/", ] # Add expected entity files From d1a5df891ef56ca4ea15bfa08d6806cc9901c5fb Mon Sep 17 00:00:00 2001 From: Daniel Wong Date: Thu, 7 Aug 2025 16:57:30 -0600 Subject: [PATCH 2/2] test: use get_entities approach in tests and improve comments --- .../apps/authoring/backup_restore/zipper.py | 43 +++++++++++-------- .../authoring/backup_restore/test_backup.py | 22 +++++++--- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/openedx_learning/apps/authoring/backup_restore/zipper.py b/openedx_learning/apps/authoring/backup_restore/zipper.py index 8175779a0..c71409fd8 100644 --- a/openedx_learning/apps/authoring/backup_restore/zipper.py +++ b/openedx_learning/apps/authoring/backup_restore/zipper.py @@ -20,13 +20,13 @@ class LearningPackageZipper: def __init__(self, learning_package: LearningPackage): self.learning_package = learning_package - def create_folder(self, folder_name: Path, zip_file: zipfile.ZipFile) -> None: + def create_folder(self, folder_path: Path, zip_file: zipfile.ZipFile) -> None: """ Create a folder for the zip file structure. Args: - folder_name (Path): The path of the folder to create. + folder_path (Path): The path of the folder to create. """ - zip_info = zipfile.ZipInfo(str(folder_name) + "/") + zip_info = zipfile.ZipInfo(str(folder_path) + "/") zip_file.writestr(zip_info, "") # Add explicit empty directory entry def create_zip(self, path: str) -> None: @@ -46,24 +46,33 @@ def create_zip(self, path: str) -> None: # Add the entities directory entities_folder = Path("entities") - zip_info = zipfile.ZipInfo(str(entities_folder) + "/") # Ensure trailing slash - zipf.writestr(zip_info, "") # Add explicit empty directory entry + self.create_folder(entities_folder, zipf) # Add the collections directory collections_folder = Path("collections") - collections_info = zipfile.ZipInfo(str(collections_folder) + "/") # Ensure trailing slash - zipf.writestr(collections_info, "") # Add explicit empty directory + self.create_folder(collections_folder, zipf) # Add each entity's TOML file for entity in publishing_api.get_entities(self.learning_package.pk): + # entity: PublishableEntity = entity # Type hint for clarity + # Create a TOML representation of the entity entity_toml_content: str = toml_publishable_entity(entity) entity_toml_filename = f"{entity.key}.toml" entity_toml_path = entities_folder / entity_toml_filename zipf.writestr(str(entity_toml_path), entity_toml_content) - # Create component directories if they exist if hasattr(entity, 'component'): + # Create the component folder structure for the entity. The structure is as follows: + # entities/ + # xblock.v1/ (component namespace) + # html/ (component type) + # my_component.toml (entity TOML file) + # my_component/ (component id) + # component_versions/ + # v1/ + # static/ + component_namespace_folder = entities_folder / entity.component.component_type.namespace # Example of component namespace is: "xblock.v1" if component_namespace_folder not in folders_already_created: @@ -71,23 +80,23 @@ def create_zip(self, path: str) -> None: folders_already_created.add(component_namespace_folder) component_type_folder = component_namespace_folder / entity.component.component_type.name - # Example of component name is: "html" + # Example of component type is: "html" if component_type_folder not in folders_already_created: self.create_folder(component_type_folder, zipf) folders_already_created.add(component_type_folder) - component_name_folder = component_type_folder / entity.component.local_key # entity.key - # Example of component name is: "i-dont-like-the-sidebar-aa1645ade4a7" - if component_name_folder not in folders_already_created: - self.create_folder(component_name_folder, zipf) - folders_already_created.add(component_name_folder) + component_id_folder = component_type_folder / entity.component.local_key # entity.key + # Example of component id is: "i-dont-like-the-sidebar-aa1645ade4a7" + if component_id_folder not in folders_already_created: + self.create_folder(component_id_folder, zipf) + folders_already_created.add(component_id_folder) - # Add the entity TOML file inside the component folder as well + # Add the entity TOML file inside the component type folder as well component_entity_toml_path = component_type_folder / f"{entity.component.local_key}.toml" zipf.writestr(str(component_entity_toml_path), entity_toml_content) - # Add component version folder - component_version_folder = component_name_folder / "component_versions" + # Add component version folder into the component id folder + component_version_folder = component_id_folder / "component_versions" if component_version_folder not in folders_already_created: self.create_folder(component_version_folder, zipf) folders_already_created.add(component_version_folder) diff --git a/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py index 0b7e9ec1d..5e361847d 100644 --- a/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py +++ b/tests/openedx_learning/apps/authoring/backup_restore/test_backup.py @@ -86,8 +86,7 @@ def setUpTestData(cls): created_by=cls.user.id, ) - # components = self.learning_package.publishable_entities.all() - components = api.get_components(cls.learning_package) + components = api.get_entities(cls.learning_package) cls.all_components = components def check_toml_file(self, zip_path: Path, zip_member_name: Path, content_to_check: list): @@ -113,6 +112,15 @@ def check_zip_file_structure(self, zip_path: Path): "collections/", "entities/xblock.v1/", "entities/xblock.v1/html/", + "entities/xblock.v1/html/my_draft_example/", + "entities/xblock.v1/html/my_draft_example/component_versions/", + "entities/xblock.v1/html/my_draft_example/component_versions/v1/", + "entities/xblock.v1/html/my_draft_example/component_versions/v1/static/", + "entities/xblock.v1/problem/", + "entities/xblock.v1/problem/my_published_example/", + "entities/xblock.v1/problem/my_published_example/component_versions/", + "entities/xblock.v1/problem/my_published_example/component_versions/v1/", + "entities/xblock.v1/problem/my_published_example/component_versions/v1/static/", ] # Add expected entity files @@ -164,20 +172,22 @@ def test_lp_dump_command(self): # Check the content of the entity TOML files for entity in self.all_components: + current_draft_version = getattr(entity, "draft", None) + current_published_version = getattr(entity, "published", None) expected_content = [ '[entity]', f'uuid = "{entity.uuid}"', 'can_stand_alone = true', '[entity.draft]', - f'version_num = {entity.versioning.draft.version_num}', + f'version_num = {current_draft_version.version.version_num}', '[entity.published]', ] - if entity.versioning.published: - expected_content.append(f'version_num = {entity.versioning.published.version_num}') + if current_published_version: + expected_content.append(f'version_num = {current_published_version.version.version_num}') else: expected_content.append('# unpublished: no published_version_num') - for entity_version in entity.versioning.versions.all(): + for entity_version in entity.versions.all(): expected_content.append(f'title = "{entity_version.title}"') expected_content.append(f'uuid = "{entity_version.uuid}"') expected_content.append(f'version_num = {entity_version.version_num}')