From 54bcacefb17e5fa1df6bbc47f8449d3a8b841ff2 Mon Sep 17 00:00:00 2001 From: Simon Harrer Date: Wed, 8 Apr 2026 11:21:36 +0200 Subject: [PATCH] Add import command for organization exports, bump version to 0.2.6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New `entropy-data import zip` command that imports an organization export zip file. Handles dependency ordering (teams → tags → definitions → assets → data contracts → data products → access), topological sorting for nested team hierarchies, and automatic member stripping. --- pyproject.toml | 2 +- src/entropy_data_cli/cli.py | 2 + .../commands/import_export.py | 134 ++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 src/entropy_data_cli/commands/import_export.py diff --git a/pyproject.toml b/pyproject.toml index aa7f504..db7e80a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "entropy-data-cli" -version = "0.2.5" +version = "0.2.6" description = "CLI for Entropy Data" requires-python = ">=3.12" license = "MIT" diff --git a/src/entropy_data_cli/cli.py b/src/entropy_data_cli/cli.py index 1b34364..f3bd3c7 100644 --- a/src/entropy_data_cli/cli.py +++ b/src/entropy_data_cli/cli.py @@ -100,6 +100,7 @@ def main( from entropy_data_cli.commands.dataproducts import dataproducts_app # noqa: E402 from entropy_data_cli.commands.definitions import definitions_app # noqa: E402 from entropy_data_cli.commands.events import events_app # noqa: E402 +from entropy_data_cli.commands.import_export import import_app # noqa: E402 from entropy_data_cli.commands.example_data import example_data_app # noqa: E402 from entropy_data_cli.commands.lineage import lineage_app # noqa: E402 from entropy_data_cli.commands.search import search_app # noqa: E402 @@ -129,3 +130,4 @@ def main( app.add_typer(lineage_app, name="lineage", help="Manage lineage (OpenLineage events).") app.add_typer(search_app, name="search", help="Search across resources.") app.add_typer(usage_app, name="usage", help="Manage usage (OpenTelemetry traces).") +app.add_typer(import_app, name="import", help="Import organization exports.") diff --git a/src/entropy_data_cli/commands/import_export.py b/src/entropy_data_cli/commands/import_export.py new file mode 100644 index 0000000..adacfaa --- /dev/null +++ b/src/entropy_data_cli/commands/import_export.py @@ -0,0 +1,134 @@ +"""Import command for organization exports.""" + +import tempfile +import zipfile +from pathlib import Path +from typing import Annotated + +import typer +import yaml + +from entropy_data_cli.client import ApiError +from entropy_data_cli.output import console, error_console + +import_app = typer.Typer(no_args_is_help=True) + +# Import order respects resource dependencies: +# teams (parent→child), tags (→teams), definitions (→teams), +# assets (→teams), datacontracts (→teams, assets), +# dataproducts (→teams, datacontracts, assets), access (→dataproducts) +RESOURCE_ORDER = [ + ("teams", "teams"), + ("tags", "tags"), + ("definitions", "definitions"), + ("assets", "assets"), + ("datacontracts", "datacontracts"), + ("dataproducts", "dataproducts"), + ("access", "access"), +] + + +def _import_teams(teams_dir: Path, client) -> tuple[int, int]: + """Import teams in topological order (parents before children), stripping members.""" + teams = {} + for f in sorted(teams_dir.glob("*.yaml")): + data = yaml.safe_load(f.read_text()) + data["members"] = [] + teams[data["id"]] = {"data": data, "parent": data.get("parent")} + + imported: set[str] = set() + success_count = 0 + error_count = 0 + + while len(imported) < len(teams): + progress = False + for tid, t in teams.items(): + if tid in imported: + continue + if t["parent"] is None or t["parent"] in imported: + try: + client.put_resource("teams", tid, t["data"]) + console.print(f" [green]OK[/green] {tid}") + success_count += 1 + except ApiError as e: + error_console.print(f" [red]FAIL[/red] {tid}: {e}") + error_count += 1 + imported.add(tid) + progress = True + + if not progress: + remaining = set(teams.keys()) - imported + error_console.print(f" [red]ERROR: circular or broken parent references: {remaining}[/red]") + error_count += len(remaining) + break + + return success_count, error_count + + +def _import_simple(resource_dir: Path, api_path: str, client) -> tuple[int, int]: + """Import resources from a directory using PUT.""" + success_count = 0 + error_count = 0 + + for f in sorted(resource_dir.glob("*.yaml")): + data = yaml.safe_load(f.read_text()) + resource_id = data["id"] + try: + client.put_resource(api_path, resource_id, data) + console.print(f" [green]OK[/green] {resource_id}") + success_count += 1 + except ApiError as e: + error_console.print(f" [red]FAIL[/red] {resource_id}: {e}") + error_count += 1 + + return success_count, error_count + + +@import_app.command("zip") +def import_zip( + file: Annotated[Path, typer.Argument(help="Path to the export zip file.")], +) -> None: + """Import an organization export zip file.""" + from entropy_data_cli.cli import get_client, handle_error + + if not file.is_file(): + error_console.print(f"[red]Error: {file} not found[/red]") + raise typer.Exit(1) + + if not zipfile.is_zipfile(file): + error_console.print(f"[red]Error: {file} is not a valid zip file[/red]") + raise typer.Exit(1) + + try: + client = get_client() + except Exception as e: + handle_error(e) + return + + with tempfile.TemporaryDirectory() as tmpdir: + export_dir = Path(tmpdir) + console.print(f"Extracting {file}...") + with zipfile.ZipFile(file) as zf: + zf.extractall(export_dir) + + total_ok = 0 + total_fail = 0 + + for directory, api_path in RESOURCE_ORDER: + resource_dir = export_dir / directory + if not resource_dir.is_dir(): + continue + + console.print(f"\n[bold]{api_path}[/bold]") + + if directory == "teams": + ok, fail = _import_teams(resource_dir, client) + else: + ok, fail = _import_simple(resource_dir, api_path, client) + + total_ok += ok + total_fail += fail + + console.print(f"\n[bold]Summary:[/bold] {total_ok} succeeded, {total_fail} failed") + if total_fail > 0: + raise typer.Exit(1)