From c29293d5c5f271c206370f5fc8abc12e8df9a6f7 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 16:24:11 +0000 Subject: [PATCH 01/16] Introduce superproject class --- dfetch/commands/check.py | 13 +++++----- dfetch/commands/diff.py | 22 ++++++++-------- dfetch/commands/freeze.py | 15 +++++++---- dfetch/commands/report.py | 9 ++++--- dfetch/commands/update.py | 12 +++++---- dfetch/manifest/manifest.py | 10 -------- dfetch/project/superproject.py | 46 ++++++++++++++++++++++++++++++++++ tests/test_check.py | 9 ++++--- tests/test_report.py | 9 ++++--- tests/test_update.py | 15 +++++++---- 10 files changed, 107 insertions(+), 53 deletions(-) create mode 100644 dfetch/project/superproject.py diff --git a/dfetch/commands/check.py b/dfetch/commands/check.py index 6b7d209ab..05794e0f3 100644 --- a/dfetch/commands/check.py +++ b/dfetch/commands/check.py @@ -31,12 +31,11 @@ import os import dfetch.commands.command -import dfetch.manifest.manifest -import dfetch.manifest.validate import dfetch.project from dfetch.commands.common import check_child_manifests, files_to_ignore from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest +from dfetch.project.superproject import SuperProject from dfetch.reporting.check.code_climate_reporter import CodeClimateReporter from dfetch.reporting.check.jenkins_reporter import JenkinsReporter from dfetch.reporting.check.reporter import CheckReporter @@ -91,12 +90,12 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, args: argparse.Namespace) -> None: """Perform the check.""" - manifest = dfetch.manifest.manifest.get_manifest() - reporters = self._get_reporters(args, manifest) + superproject = SuperProject() + reporters = self._get_reporters(args, superproject.manifest) - with in_directory(os.path.dirname(manifest.path)): + with in_directory(superproject.root_directory): exceptions: list[str] = [] - for project in manifest.selected_projects(args.projects): + for project in superproject.manifest.selected_projects(args.projects): with catch_runtime_exceptions(exceptions) as exceptions: dfetch.project.make(project).check_for_update( reporters, files_to_ignore=files_to_ignore(project.destination) @@ -104,7 +103,7 @@ def __call__(self, args: argparse.Namespace) -> None: if not args.no_recommendations and os.path.isdir(project.destination): with in_directory(project.destination): - check_child_manifests(manifest, project) + check_child_manifests(superproject.manifest, project) for reporter in reporters: reporter.dump_to_file() diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index 7879bf33d..4c44bb24b 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -97,13 +97,12 @@ import os import dfetch.commands.command -import dfetch.manifest.manifest -import dfetch.manifest.validate import dfetch.project from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.project.git import GitRepo from dfetch.project.metadata import Metadata +from dfetch.project.superproject import SuperProject from dfetch.project.svn import SvnRepo from dfetch.project.vcs import VCS from dfetch.util.util import catch_runtime_exceptions, in_directory @@ -142,12 +141,12 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, args: argparse.Namespace) -> None: """Perform the diff.""" - manifest = dfetch.manifest.manifest.get_manifest() + superproject = SuperProject() revs = [r for r in args.revs.strip(":").split(":", maxsplit=1) if r] - with in_directory(os.path.dirname(manifest.path)): + with in_directory(superproject.root_directory): exceptions: list[str] = [] - projects = manifest.selected_projects(args.projects) + projects = superproject.manifest.selected_projects(args.projects) if not projects: raise RuntimeError( f"No (such) project found! {', '.join(args.projects)}" @@ -155,25 +154,26 @@ def __call__(self, args: argparse.Namespace) -> None: for project in projects: patch_name = f"{project.name}.patch" with catch_runtime_exceptions(exceptions) as exceptions: - repo = _get_repo(manifest.path, project) + repo = _get_repo(superproject, project) patch = _diff_from_repo(repo, project, revs) - _dump_patch(manifest.path, revs, project, patch_name, patch) + _dump_patch( + superproject.manifest.path, revs, project, patch_name, patch + ) if exceptions: raise RuntimeError("\n".join(exceptions)) -def _get_repo(path: str, project: ProjectEntry) -> VCS: +def _get_repo(superproject: SuperProject, project: ProjectEntry) -> VCS: """Get the repo type from the project.""" if not os.path.exists(project.destination): raise RuntimeError( "You cannot generate a diff of a project that was never fetched" ) - main_project_dir = os.path.dirname(path) - if GitLocalRepo(main_project_dir).is_git(): + if GitLocalRepo(superproject.root_directory).is_git(): return GitRepo(project) - if SvnRepo.check_path(main_project_dir): + if SvnRepo.check_path(superproject.root_directory): return SvnRepo(project) raise RuntimeError( diff --git a/dfetch/commands/freeze.py b/dfetch/commands/freeze.py index 0eb648c8a..8b8b17d9d 100644 --- a/dfetch/commands/freeze.py +++ b/dfetch/commands/freeze.py @@ -47,8 +47,9 @@ import dfetch.project from dfetch import DEFAULT_MANIFEST_NAME from dfetch.log import get_logger -from dfetch.manifest.manifest import Manifest, get_manifest +from dfetch.manifest.manifest import Manifest from dfetch.manifest.project import ProjectEntry +from dfetch.project.superproject import SuperProject from dfetch.util.util import catch_runtime_exceptions, in_directory logger = get_logger(__name__) @@ -69,13 +70,13 @@ def __call__(self, args: argparse.Namespace) -> None: """Perform the freeze.""" del args # unused - manifest = get_manifest() + superproject = SuperProject() exceptions: list[str] = [] projects: list[ProjectEntry] = [] - with in_directory(os.path.dirname(manifest.path)): - for project in manifest.projects: + with in_directory(superproject.root_directory): + for project in superproject.manifest.projects: with catch_runtime_exceptions(exceptions) as exceptions: on_disk_version = dfetch.project.make(project).on_disk_version() @@ -98,7 +99,11 @@ def __call__(self, args: argparse.Namespace) -> None: projects.append(project) manifest = Manifest( - {"version": "0.0", "remotes": manifest.remotes, "projects": projects} + { + "version": "0.0", + "remotes": superproject.manifest.remotes, + "projects": projects, + } ) shutil.move(DEFAULT_MANIFEST_NAME, DEFAULT_MANIFEST_NAME + ".backup") diff --git a/dfetch/commands/report.py b/dfetch/commands/report.py index 07f0b2e8c..f7b698040 100644 --- a/dfetch/commands/report.py +++ b/dfetch/commands/report.py @@ -13,6 +13,7 @@ from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.project.metadata import Metadata +from dfetch.project.superproject import SuperProject from dfetch.project.vcs import VCS from dfetch.reporting import REPORTERS, ReportTypes from dfetch.util.license import License, guess_license_in_file @@ -62,12 +63,12 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, args: argparse.Namespace) -> None: """Generate the report.""" - manifest = dfetch.manifest.manifest.get_manifest() + superproject = SuperProject() - with dfetch.util.util.in_directory(os.path.dirname(manifest.path)): - reporter = REPORTERS[args.type](manifest) + with dfetch.util.util.in_directory(superproject.root_directory): + reporter = REPORTERS[args.type](superproject.manifest) - for project in manifest.selected_projects(args.projects): + for project in superproject.manifest.selected_projects(args.projects): determined_licenses = self._determine_licenses(project) version = self._determine_version(project) reporter.add_project( diff --git a/dfetch/commands/update.py b/dfetch/commands/update.py index 387b8ad10..22cf5c85d 100644 --- a/dfetch/commands/update.py +++ b/dfetch/commands/update.py @@ -39,6 +39,7 @@ import dfetch.project.svn from dfetch.commands.common import check_child_manifests, files_to_ignore from dfetch.log import get_logger +from dfetch.project.superproject import SuperProject from dfetch.util.util import catch_runtime_exceptions, in_directory logger = get_logger(__name__) @@ -76,14 +77,15 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, args: argparse.Namespace) -> None: """Perform the update.""" - manifest = dfetch.manifest.manifest.get_manifest() + superproject = SuperProject() exceptions: list[str] = [] destinations: list[str] = [ - os.path.realpath(project.destination) for project in manifest.projects + os.path.realpath(project.destination) + for project in superproject.manifest.projects ] - with in_directory(os.path.dirname(manifest.path)): - for project in manifest.selected_projects(args.projects): + with in_directory(superproject.root_directory): + for project in superproject.manifest.selected_projects(args.projects): with catch_runtime_exceptions(exceptions) as exceptions: self._check_destination(project, destinations) dfetch.project.make(project).update( @@ -95,7 +97,7 @@ def __call__(self, args: argparse.Namespace) -> None: project.destination ): with in_directory(project.destination): - check_child_manifests(manifest, project) + check_child_manifests(superproject.manifest, project) if exceptions: raise RuntimeError("\n".join(exceptions)) diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py index dc679d364..1e329cb77 100644 --- a/dfetch/manifest/manifest.py +++ b/dfetch/manifest/manifest.py @@ -376,16 +376,6 @@ def find_manifest() -> str: return os.path.realpath(paths[0]) -def get_manifest() -> Manifest: - """Get manifest and its path.""" - logger.debug("Looking for manifest") - manifest_path = find_manifest() - validate(manifest_path) - - logger.debug(f"Using manifest {manifest_path}") - return Manifest.from_file(manifest_path) - - def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]: """Get manifest and its path.""" skip = skip or [] diff --git a/dfetch/project/superproject.py b/dfetch/project/superproject.py new file mode 100644 index 000000000..4b9f59fc7 --- /dev/null +++ b/dfetch/project/superproject.py @@ -0,0 +1,46 @@ +"""Super project abstraction. + +This module provides the SuperProject class which represents the project that +contains the `dfetch.yaml` manifest file (the "super project"). It provides +helpers to query VCS information about that repository (for example whether +it's a git or svn repository). +""" + +from __future__ import annotations + +import os + +from dfetch.log import get_logger +from dfetch.manifest.manifest import Manifest, find_manifest +from dfetch.manifest.validate import validate + +logger = get_logger(__name__) + + +class SuperProject: + """Representation of the project containing the manifest. + + A SuperProject is the repository/directory that contains the dfetch + manifest file. It exposes helpers to determine whether that project is + managed by git, svn, or is unversioned. + """ + + def __init__(self) -> None: + """Create a SuperProject by looking for a manifest file.""" + logger.debug("Looking for manifest") + manifest_path = find_manifest() + validate(manifest_path) + + logger.debug(f"Using manifest {manifest_path}") + self._manifest = Manifest.from_file(manifest_path) + self._root_directory = os.path.dirname(self._manifest.path) + + @property + def root_directory(self) -> str: + """Return the directory that contains the manifest file.""" + return self._root_directory + + @property + def manifest(self) -> Manifest: + """The manifest of the super project.""" + return self._manifest diff --git a/tests/test_check.py b/tests/test_check.py index 075e1f5ba..d67a0559f 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -4,7 +4,7 @@ # flake8: noqa import argparse -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -28,7 +28,11 @@ def test_check(name, projects): check = Check() - with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: + fake_superproject = Mock() + fake_superproject.manifest = mock_manifest(projects) + fake_superproject.root_directory = "/tmp" + + with patch("dfetch.commands.check.SuperProject", return_value=fake_superproject): with patch( "dfetch.manifest.manifest.get_childmanifests" ) as mocked_get_childmanifests: @@ -36,7 +40,6 @@ def test_check(name, projects): with patch("os.path.exists"): with patch("dfetch.commands.check.in_directory"): with patch("dfetch.commands.check.CheckStdoutReporter"): - mocked_get_manifest.return_value = mock_manifest(projects) mocked_get_childmanifests.return_value = [] check(DEFAULT_ARGS) diff --git a/tests/test_report.py b/tests/test_report.py index 635839e4d..9ac2cebc7 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -4,7 +4,7 @@ # flake8: noqa import argparse -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -28,9 +28,12 @@ def test_report(name, projects): report = Report() - with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: + fake_superproject = Mock() + fake_superproject.manifest = mock_manifest(projects) + fake_superproject.root_directory = "/tmp" + + with patch("dfetch.commands.report.SuperProject", return_value=fake_superproject): with patch("dfetch.log.DLogger.print_info_line") as mocked_print_info_line: - mocked_get_manifest.return_value = mock_manifest(projects) report(DEFAULT_ARGS) diff --git a/tests/test_update.py b/tests/test_update.py index b86c88fa9..7f61c985e 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -4,7 +4,7 @@ # flake8: noqa import argparse -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -28,7 +28,11 @@ def test_update(name, projects): update = Update() - with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: + fake_superproject = Mock() + fake_superproject.manifest = mock_manifest(projects) + fake_superproject.root_directory = "/tmp" + + with patch("dfetch.commands.update.SuperProject", return_value=fake_superproject): with patch( "dfetch.manifest.manifest.get_childmanifests" ) as mocked_get_childmanifests: @@ -36,7 +40,6 @@ def test_update(name, projects): with patch("os.path.exists"): with patch("dfetch.commands.update.in_directory"): with patch("dfetch.commands.update.Update._check_destination"): - mocked_get_manifest.return_value = mock_manifest(projects) mocked_get_childmanifests.return_value = [] update(DEFAULT_ARGS) @@ -48,9 +51,11 @@ def test_update(name, projects): def test_forced_update(): update = Update() - with patch("dfetch.manifest.manifest.get_manifest") as mocked_get_manifest: - mocked_get_manifest.return_value = mock_manifest([{"name": "some_project"}]) + fake_superproject = Mock() + fake_superproject.manifest = mock_manifest([{"name": "some_project"}]) + fake_superproject.root_directory = "/tmp" + with patch("dfetch.commands.update.SuperProject", return_value=fake_superproject): with patch( "dfetch.manifest.manifest.get_childmanifests" ) as mocked_get_childmanifests: From 5857066da56ab91f922910d7a29bddd88c672e33 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 19:59:01 +0000 Subject: [PATCH 02/16] rename vcs to subproject --- dfetch/commands/diff.py | 6 ++-- dfetch/commands/report.py | 4 +-- dfetch/project/__init__.py | 6 ++-- dfetch/project/git.py | 8 ++--- dfetch/project/{vcs.py => subproject.py} | 21 ++++++------- dfetch/project/svn.py | 8 ++--- .../uml/c3_dfetch_components_project.puml | 18 +++++------ tests/{test_vcs.py => test_subproject.py} | 30 +++++++++++-------- 8 files changed, 53 insertions(+), 48 deletions(-) rename dfetch/project/{vcs.py => subproject.py} (95%) rename tests/{test_vcs.py => test_subproject.py} (79%) diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index 4c44bb24b..b84b5eed3 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -102,9 +102,9 @@ from dfetch.manifest.project import ProjectEntry from dfetch.project.git import GitRepo from dfetch.project.metadata import Metadata +from dfetch.project.subproject import SubProject from dfetch.project.superproject import SuperProject from dfetch.project.svn import SvnRepo -from dfetch.project.vcs import VCS from dfetch.util.util import catch_runtime_exceptions, in_directory from dfetch.vcs.git import GitLocalRepo @@ -165,7 +165,7 @@ def __call__(self, args: argparse.Namespace) -> None: raise RuntimeError("\n".join(exceptions)) -def _get_repo(superproject: SuperProject, project: ProjectEntry) -> VCS: +def _get_repo(superproject: SuperProject, project: ProjectEntry) -> SubProject: """Get the repo type from the project.""" if not os.path.exists(project.destination): raise RuntimeError( @@ -181,7 +181,7 @@ def _get_repo(superproject: SuperProject, project: ProjectEntry) -> VCS: ) -def _diff_from_repo(repo: VCS, project: ProjectEntry, revs: list[str]) -> str: +def _diff_from_repo(repo: SubProject, project: ProjectEntry, revs: list[str]) -> str: """Generate a relative diff for a svn repo.""" if len(revs) > 2: raise RuntimeError(f"Too many revisions given! {revs}") diff --git a/dfetch/commands/report.py b/dfetch/commands/report.py index f7b698040..82296f7e2 100644 --- a/dfetch/commands/report.py +++ b/dfetch/commands/report.py @@ -13,8 +13,8 @@ from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.project.metadata import Metadata +from dfetch.project.subproject import SubProject from dfetch.project.superproject import SuperProject -from dfetch.project.vcs import VCS from dfetch.reporting import REPORTERS, ReportTypes from dfetch.util.license import License, guess_license_in_file @@ -90,7 +90,7 @@ def _determine_licenses(project: ProjectEntry) -> list[License]: license_files = [] with dfetch.util.util.in_directory(project.destination): - for license_file in filter(VCS.is_license_file, glob.glob("*")): + for license_file in filter(SubProject.is_license_file, glob.glob("*")): logger.debug(f"Found license file {license_file} for {project.name}") guessed_license = guess_license_in_file(license_file) diff --git a/dfetch/project/__init__.py b/dfetch/project/__init__.py index 73912a4bd..fc35a1c00 100644 --- a/dfetch/project/__init__.py +++ b/dfetch/project/__init__.py @@ -2,14 +2,14 @@ import dfetch.manifest.project from dfetch.project.git import GitRepo +from dfetch.project.subproject import SubProject from dfetch.project.svn import SvnRepo -from dfetch.project.vcs import VCS SUPPORTED_PROJECT_TYPES = [GitRepo, SvnRepo] -def make(project_entry: dfetch.manifest.project.ProjectEntry) -> VCS: - """Create a new VCS based on a project from the manifest.""" +def make(project_entry: dfetch.manifest.project.ProjectEntry) -> SubProject: + """Create a new SubProject based on a project from the manifest.""" for project_type in SUPPORTED_PROJECT_TYPES: if project_type.NAME == project_entry.vcs: return project_type(project_entry) diff --git a/dfetch/project/git.py b/dfetch/project/git.py index b4f722a28..043b25224 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -9,14 +9,14 @@ from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version -from dfetch.project.vcs import VCS +from dfetch.project.subproject import SubProject from dfetch.util.util import safe_rmtree from dfetch.vcs.git import GitLocalRepo, GitRemote, get_git_version logger = get_logger(__name__) -class GitRepo(VCS): +class GitRepo(SubProject): """A git repository.""" NAME = "git" @@ -83,12 +83,12 @@ def list_tool_info() -> None: """Print out version information.""" try: tool, version = get_git_version() - VCS._log_tool(tool, version) + SubProject._log_tool(tool, version) except RuntimeError as exc: logger.debug( f"Something went wrong trying to get the version of git: {exc}" ) - VCS._log_tool("git", "") + SubProject._log_tool("git", "") def _fetch_impl(self, version: Version) -> Version: """Get the revision of the remote and place it at the local path.""" diff --git a/dfetch/project/vcs.py b/dfetch/project/subproject.py similarity index 95% rename from dfetch/project/vcs.py rename to dfetch/project/subproject.py index 73d6c397f..a6f008da1 100644 --- a/dfetch/project/vcs.py +++ b/dfetch/project/subproject.py @@ -21,8 +21,8 @@ logger = get_logger(__name__) -class VCS(ABC): - """Abstract Version Control System object. +class SubProject(ABC): + """Abstract SubProject object. This object represents one Project entry in the Manifest. It can be updated. @@ -32,7 +32,7 @@ class VCS(ABC): LICENSE_GLOBS = ["licen[cs]e*", "copying*", "copyright*"] def __init__(self, project: ProjectEntry) -> None: - """Create the VCS.""" + """Create the subproject.""" self.__project = project self.__metadata = Metadata.from_project_entry(self.__project) @@ -95,7 +95,7 @@ def update_is_required(self, force: bool = False) -> Optional[Version]: def update( self, force: bool = False, files_to_ignore: Optional[Sequence[str]] = None ) -> None: - """Update this VCS if required. + """Update this subproject if required. Args: force (bool, optional): Ignore if version is ok or any local changes were done. @@ -230,7 +230,7 @@ def local_path(self) -> str: @property def wanted_version(self) -> Version: - """Get the wanted version of this VCS.""" + """Get the wanted version of this subproject.""" return self.__metadata.version @property @@ -240,17 +240,17 @@ def metadata_path(self) -> str: @property def remote(self) -> str: - """Get the remote URL of this VCS.""" + """Get the remote URL of this subproject.""" return self.__metadata.remote_url @property def source(self) -> str: - """Get the source folder of this VCS.""" + """Get the source folder of this subproject.""" return self.__project.source @property def ignore(self) -> Sequence[str]: - """Get the files/folders to ignore of this VCS.""" + """Get the files/folders to ignore of this subproject.""" return self.__project.ignore @abstractmethod @@ -362,7 +362,7 @@ def _are_there_local_changes(self, files_to_ignore: Sequence[str]) -> bool: @abstractmethod def _fetch_impl(self, version: Version) -> Version: - """Fetch the given version of the VCS, should be implemented by the child class.""" + """Fetch the given version of the subproject, should be implemented by the child class.""" @abstractmethod def metadata_revision(self) -> str: @@ -386,5 +386,6 @@ def get_default_branch(self) -> str: def is_license_file(filename: str) -> bool: """Check if the given filename is a license file.""" return any( - fnmatch.fnmatch(filename.lower(), pattern) for pattern in VCS.LICENSE_GLOBS + fnmatch.fnmatch(filename.lower(), pattern) + for pattern in SubProject.LICENSE_GLOBS ) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 09f8824ec..b5a35ec54 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -9,7 +9,7 @@ from dfetch.log import get_logger from dfetch.manifest.version import Version -from dfetch.project.vcs import VCS +from dfetch.project.subproject import SubProject from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline from dfetch.util.util import ( find_matching_files, @@ -39,7 +39,7 @@ class External(NamedTuple): src: str -class SvnRepo(VCS): +class SvnRepo(SubProject): """A svn repository.""" DEFAULT_BRANCH = "trunk" @@ -188,12 +188,12 @@ def list_tool_info() -> None: logger.debug( f"Something went wrong trying to get the version of svn: {exc}" ) - VCS._log_tool("svn", "") + SubProject._log_tool("svn", "") return first_line = result.stdout.decode().split("\n")[0] tool, version = first_line.replace(",", "").split("version", maxsplit=1) - VCS._log_tool(tool, version) + SubProject._log_tool(tool, version) def _determine_what_to_fetch(self, version: Version) -> tuple[str, str, str]: """Based on the given version, determine what to fetch. diff --git a/doc/static/uml/c3_dfetch_components_project.puml b/doc/static/uml/c3_dfetch_components_project.puml index 4c986b07f..0c99edbc8 100644 --- a/doc/static/uml/c3_dfetch_components_project.puml +++ b/doc/static/uml/c3_dfetch_components_project.puml @@ -12,28 +12,28 @@ System_Boundary(DFetch, "Dfetch") { Boundary(DfetchProject, "Project", "python", "Main project that has a manifest.") { Component(compAbstractCheckReporter, "AbstractCheckReporter", "python", "Abstract interface for generating a check report.") - Component(compGit, "Git", "python", "A remote source project based on git.") + Component(compGit, "Git", "python", "A subproject based on git.") Component(compMetadata, "Metadata", "python", "A file containing metadata about a project.") - Component(compSvn, "Svn", "python", "A remote source project based on svn.") - Component(compVcs, "Vcs", "python", "An abstract remote version control system.") + Component(compSvn, "Svn", "python", "A subproject based on svn.") + Component(compSubProject, "SubProject", "python", "An abstract subproject.") - Rel_U(compGit, compVcs, "Implements") - Rel_U(compSvn, compVcs, "Implements") - Rel(compVcs, compAbstractCheckReporter, "Uses") - Rel_L(compVcs, compMetadata, "Uses") + Rel_U(compGit, compSubProject, "Implements") + Rel_U(compSvn, compSubProject, "Implements") + Rel(compSubProject, compAbstractCheckReporter, "Uses") + Rel_L(compSubProject, compMetadata, "Uses") } Container(contVcs, "Vcs", "python", "Abstraction of various Version Control Systems.") Container(contReporting, "Reporting", "python", "Output formatters for various reporting formats.") - Rel(contCommands, compVcs, "Uses") + Rel(contCommands, compSubProject, "Uses") Rel(contCommands, compGit, "Uses") Rel(contCommands, compSvn, "Uses") Rel(contReporting, compAbstractCheckReporter, "Implements") Rel_R(contReporting, compMetadata, "Uses") Rel_R(compMetadata, contManifest, "Has") - Rel(compVcs, contManifest, "Has") + Rel(compSubProject, contManifest, "Has") Rel(contReporting, contManifest, "Uses") Rel(compGit, contVcs, "Uses") Rel(compSvn, contVcs, "Uses") diff --git a/tests/test_vcs.py b/tests/test_subproject.py similarity index 79% rename from tests/test_vcs.py rename to tests/test_subproject.py index ff7c9a49e..ce0079be3 100644 --- a/tests/test_vcs.py +++ b/tests/test_subproject.py @@ -1,4 +1,4 @@ -"""Test the vcs class.""" +"""Test the subproject class.""" # mypy: ignore-errors # flake8: noqa @@ -10,10 +10,10 @@ from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version -from dfetch.project.vcs import VCS +from dfetch.project.subproject import SubProject -class ConcreteVCS(VCS): +class ConcreteSubProject(SubProject): _wanted_version: Version def _fetch_impl(self, version: Version) -> Version: @@ -104,16 +104,16 @@ def test_check_wanted_with_local( expect_wanted: Version, expect_have: Union[Version, None], ): - with patch("dfetch.project.vcs.os.path.exists") as mocked_path_exists: - with patch("dfetch.project.vcs.Metadata.from_file") as mocked_metadata: - vcs = ConcreteVCS(ProjectEntry({"name": "proj1"})) + with patch("dfetch.project.subproject.os.path.exists") as mocked_path_exists: + with patch("dfetch.project.subproject.Metadata.from_file") as mocked_metadata: + subproject = ConcreteSubProject(ProjectEntry({"name": "proj1"})) mocked_path_exists.return_value = bool(given_on_disk) mocked_metadata().version = given_on_disk - vcs._wanted_version = given_wanted + subproject._wanted_version = given_wanted - wanted, have = vcs.check_wanted_with_local() + wanted, have = subproject.check_wanted_with_local() assert wanted == expect_wanted assert have == expect_have @@ -130,14 +130,18 @@ def test_check_wanted_with_local( def test_are_there_local_changes( name: str, hash_in_metadata: str, current_hash: str, expectation: bool ): - with patch("dfetch.project.vcs.hash_directory") as mocked_hash_directory: - with patch("dfetch.project.vcs.VCS._on_disk_hash") as mocked_on_disk_hash: - vcs = ConcreteVCS(ProjectEntry({"name": "proj1"})) + with patch("dfetch.project.subproject.hash_directory") as mocked_hash_directory: + with patch( + "dfetch.project.subproject.SubProject._on_disk_hash" + ) as mocked_on_disk_hash: + subproject = ConcreteSubProject(ProjectEntry({"name": "proj1"})) mocked_on_disk_hash.return_value = hash_in_metadata mocked_hash_directory.return_value = current_hash - assert expectation == vcs._are_there_local_changes(files_to_ignore=[]) + assert expectation == subproject._are_there_local_changes( + files_to_ignore=[] + ) @pytest.mark.parametrize( @@ -162,4 +166,4 @@ def test_ci_enabled( else: monkeypatch.setenv("CI", str(ci_env_value)) - assert ConcreteVCS._running_in_ci() == expected_result + assert ConcreteSubProject._running_in_ci() == expected_result From e8827b07191cfa31c312bf9dcf46203975382357 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 20:07:38 +0000 Subject: [PATCH 03/16] Fix comment --- dfetch/project/subproject.py | 9 ++++++--- doc/internal.rst | 29 ++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index a6f008da1..e52de9e94 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -1,4 +1,4 @@ -"""Version Control system.""" +"""SubProject.""" import fnmatch import os @@ -374,8 +374,11 @@ def current_revision(self) -> str: @abstractmethod def get_diff( - self, old_revision: str, new_revision: Optional[str], ignore: Sequence[str] - ) -> str: # noqa + self, + old_revision: str, # noqa + new_revision: Optional[str], # noqa + ignore: Sequence[str], + ) -> str: """Get the diff of two revisions.""" @abstractmethod diff --git a/doc/internal.rst b/doc/internal.rst index 57bab0022..29ac27ff0 100644 --- a/doc/internal.rst +++ b/doc/internal.rst @@ -2,7 +2,34 @@ Internal ======== -*DFetch* is becoming larger everyday. To give it some structure below a description of the internals +*DFetch* is becoming larger everyday. To give it some structure below a description of the internals. + +Glossary +-------- + +Superproject + The top-level project that contains the manifest. It defines and + coordinates all included projects. + +Subproject + A project defined in the manifest that is copied into the + superproject. Subprojects are managed and updated as part of the + superproject's configuration. + +Remote + Defines a source repository base URL and a name. Remotes + allow you to avoid repeating common URL bases for multiple + projects in a manifest. A single remote may contain multiple + (sub-)projects to fetch. + +Child Manifest + Some subprojects can themselves contain a manifest. When + fetching a subproject, dfetch can optionally check these + child manifests for additional dependencies or recommendations. + +Metadata + A file created by *DFetch* to store some relevant information about + a subproject. Architecture ------------ From cfa79741f046412db1ae895c689a8a8c9da2306e Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 20:26:18 +0000 Subject: [PATCH 04/16] Rename Git & SvnRepo to SubProject --- dfetch/commands/common.py | 6 +++--- dfetch/commands/diff.py | 10 +++++----- dfetch/commands/import_.py | 6 +++--- dfetch/project/__init__.py | 6 +++--- dfetch/project/git.py | 6 +++--- dfetch/project/svn.py | 20 +++++++++++--------- tests/test_import.py | 6 ++++-- tests/test_svn.py | 14 +++++++------- 8 files changed, 39 insertions(+), 35 deletions(-) diff --git a/dfetch/commands/common.py b/dfetch/commands/common.py index af0105d78..7a1c1557e 100644 --- a/dfetch/commands/common.py +++ b/dfetch/commands/common.py @@ -8,7 +8,7 @@ from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest, get_childmanifests from dfetch.manifest.project import ProjectEntry -from dfetch.project.svn import SvnRepo +from dfetch.project.svn import SvnSubProject from dfetch.vcs.git import GitLocalRepo logger = get_logger(__name__) @@ -72,8 +72,8 @@ def files_to_ignore(path: str) -> Sequence[str]: """Return a list of files that can be ignored in a given path.""" if GitLocalRepo().is_git(): ignore_list = GitLocalRepo.ignored_files(path) - elif SvnRepo.check_path(): - ignore_list = SvnRepo.ignored_files(path) + elif SvnSubProject.check_path(): + ignore_list = SvnSubProject.ignored_files(path) else: ignore_list = [] return ignore_list diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index b84b5eed3..6fb5138e4 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -100,11 +100,11 @@ import dfetch.project from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry -from dfetch.project.git import GitRepo +from dfetch.project.git import GitSubProject from dfetch.project.metadata import Metadata from dfetch.project.subproject import SubProject from dfetch.project.superproject import SuperProject -from dfetch.project.svn import SvnRepo +from dfetch.project.svn import SvnSubProject from dfetch.util.util import catch_runtime_exceptions, in_directory from dfetch.vcs.git import GitLocalRepo @@ -172,9 +172,9 @@ def _get_repo(superproject: SuperProject, project: ProjectEntry) -> SubProject: "You cannot generate a diff of a project that was never fetched" ) if GitLocalRepo(superproject.root_directory).is_git(): - return GitRepo(project) - if SvnRepo.check_path(superproject.root_directory): - return SvnRepo(project) + return GitSubProject(project) + if SvnSubProject.check_path(superproject.root_directory): + return SvnSubProject(project) raise RuntimeError( "Can only create patch in SVN or Git repo", diff --git a/dfetch/commands/import_.py b/dfetch/commands/import_.py index f22a087f7..a35dc281c 100644 --- a/dfetch/commands/import_.py +++ b/dfetch/commands/import_.py @@ -92,7 +92,7 @@ from dfetch.manifest.manifest import Manifest from dfetch.manifest.project import ProjectEntry from dfetch.manifest.remote import Remote -from dfetch.project.svn import SvnRepo +from dfetch.project.svn import SvnSubProject from dfetch.vcs.git import GitLocalRepo logger = get_logger(__name__) @@ -139,7 +139,7 @@ def _import_projects() -> Sequence[ProjectEntry]: """Find out what type of VCS is used and import projects.""" if GitLocalRepo().is_git(): projects = _import_from_git() - elif SvnRepo.check_path(): + elif SvnSubProject.check_path(): projects = _import_from_svn() else: raise RuntimeError( @@ -152,7 +152,7 @@ def _import_projects() -> Sequence[ProjectEntry]: def _import_from_svn() -> Sequence[ProjectEntry]: projects: list[ProjectEntry] = [] - for external in SvnRepo.externals(): + for external in SvnSubProject.externals(): projects.append( ProjectEntry( { diff --git a/dfetch/project/__init__.py b/dfetch/project/__init__.py index fc35a1c00..246517563 100644 --- a/dfetch/project/__init__.py +++ b/dfetch/project/__init__.py @@ -1,11 +1,11 @@ """All Project related items.""" import dfetch.manifest.project -from dfetch.project.git import GitRepo +from dfetch.project.git import GitSubProject from dfetch.project.subproject import SubProject -from dfetch.project.svn import SvnRepo +from dfetch.project.svn import SvnSubProject -SUPPORTED_PROJECT_TYPES = [GitRepo, SvnRepo] +SUPPORTED_PROJECT_TYPES = [GitSubProject, SvnSubProject] def make(project_entry: dfetch.manifest.project.ProjectEntry) -> SubProject: diff --git a/dfetch/project/git.py b/dfetch/project/git.py index 043b25224..9542ce418 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -16,13 +16,13 @@ logger = get_logger(__name__) -class GitRepo(SubProject): - """A git repository.""" +class GitSubProject(SubProject): + """A git subproject.""" NAME = "git" def __init__(self, project: ProjectEntry): - """Create a Git project.""" + """Create a Git subproject.""" super().__init__(project) self._remote_repo = GitRemote(self.remote) self._local_repo = GitLocalRepo(self.local_path) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index b5a35ec54..81382e049 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -39,8 +39,8 @@ class External(NamedTuple): src: str -class SvnRepo(SubProject): - """A svn repository.""" +class SvnSubProject(SubProject): + """A svn subproject.""" DEFAULT_BRANCH = "trunk" NAME = "svn" @@ -59,7 +59,7 @@ def externals() -> list[External]: ], ) - repo_root = SvnRepo._get_info_from_target()["Repository Root"] + repo_root = SvnSubProject._get_info_from_target()["Repository Root"] externals: list[External] = [] path_pattern = r"([^\s^-]+)\s+-" @@ -80,7 +80,7 @@ def externals() -> list[External]: name = match.group(3) or match.group(5) rev = "" if not match.group(2) else match.group(2).strip() - url, branch, tag, src = SvnRepo._split_url(url, repo_root) + url, branch, tag, src = SvnSubProject._split_url(url, repo_root) externals += [ External( @@ -112,7 +112,9 @@ def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: r"(.*)\/(branches\/[^\/]+|tags\/[^\/]+|trunk)[\/]*(.*)", url ): url = match.group(1) - branch = match.group(2) if match.group(2) != SvnRepo.DEFAULT_BRANCH else "" + branch = ( + match.group(2) if match.group(2) != SvnSubProject.DEFAULT_BRANCH else "" + ) src = match.group(3) path = branch.split("/") @@ -252,7 +254,7 @@ def _fetch_impl(self, version: Version) -> Version: complete_path, file_pattern = self._parse_file_pattern(complete_path) - SvnRepo._export(complete_path, rev_arg, self.local_path) + SvnSubProject._export(complete_path, rev_arg, self.local_path) if file_pattern: for file in find_non_matching_files(self.local_path, (file_pattern,)): @@ -265,13 +267,13 @@ def _fetch_impl(self, version: Version) -> Version: if self.source: root_branch_path = "/".join([self.remote, branch_path]).strip("/") - for file in SvnRepo._license_files(root_branch_path): + for file in SvnSubProject._license_files(root_branch_path): dest = ( self.local_path if os.path.isdir(self.local_path) else os.path.dirname(self.local_path) ) - SvnRepo._export(f"{root_branch_path}/{file}", rev_arg, dest) + SvnSubProject._export(f"{root_branch_path}/{file}", rev_arg, dest) break if self.ignore: @@ -319,7 +321,7 @@ def _license_files(url_path: str) -> list[str]: return [ str(license) for license in filter( - SvnRepo.is_license_file, SvnRepo._files_in_path(url_path) + SvnSubProject.is_license_file, SvnSubProject._files_in_path(url_path) ) ] diff --git a/tests/test_import.py b/tests/test_import.py index 219c7488c..04d23818f 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -105,8 +105,10 @@ def test_git_import(name, submodules): def test_svn_import(name, externals): import_ = Import() - with patch("dfetch.commands.import_.SvnRepo.check_path") as check_path: - with patch("dfetch.commands.import_.SvnRepo.externals") as mocked_externals: + with patch("dfetch.commands.import_.SvnSubProject.check_path") as check_path: + with patch( + "dfetch.commands.import_.SvnSubProject.externals" + ) as mocked_externals: with patch("dfetch.commands.import_.Manifest") as mocked_manifest: with patch("dfetch.commands.import_.GitLocalRepo.is_git") as is_git: is_git.return_value = False diff --git a/tests/test_svn.py b/tests/test_svn.py index 72ea70faa..985d81faa 100644 --- a/tests/test_svn.py +++ b/tests/test_svn.py @@ -9,7 +9,7 @@ import pytest from dfetch.manifest.project import ProjectEntry -from dfetch.project.svn import External, SvnRepo +from dfetch.project.svn import External, SvnSubProject from dfetch.util.cmdline import SubprocessCommandError REPO_ROOT = "repo-root" @@ -131,7 +131,7 @@ def test_externals(name, externals, expectations): with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: with patch( - "dfetch.project.svn.SvnRepo._get_info_from_target" + "dfetch.project.svn.SvnSubProject._get_info_from_target" ) as target_info_mock: with patch("dfetch.project.svn.os.getcwd") as cwd_mock: cmd_output = str(os.linesep * 2).join(externals) @@ -139,7 +139,7 @@ def test_externals(name, externals, expectations): target_info_mock.return_value = {"Repository Root": REPO_ROOT} cwd_mock.return_value = CWD - parsed_externals = SvnRepo.externals() + parsed_externals = SvnSubProject.externals() for actual, expected in zip( parsed_externals, expectations # , strict=True @@ -159,7 +159,7 @@ def test_check_path(name, cmd_result, expectation): with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result - assert SvnRepo.check_path() == expectation + assert SvnSubProject.check_path() == expectation @pytest.mark.parametrize( @@ -184,7 +184,7 @@ def test_check(name, project, cmd_result, expectation): with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result - assert SvnRepo(project).check() == expectation + assert SvnSubProject(project).check() == expectation SVN_INFO = """ @@ -207,7 +207,7 @@ def test_get_info(): run_on_cmdline_mock.return_value.stdout = os.linesep.join( SVN_INFO.split("\n") ).encode() - result = SvnRepo._get_info_from_target("bla") + result = SvnSubProject._get_info_from_target("bla") expectation = { "Path": "cpputest", @@ -227,7 +227,7 @@ def test_get_info(): @pytest.fixture def svn_repo(): - return SvnRepo(ProjectEntry({"name": "proj3", "url": "some_url"})) + return SvnSubProject(ProjectEntry({"name": "proj3", "url": "some_url"})) def test_svn_repo_name(svn_repo): From 7890deaeb80a4bb8461a96e8d066b7a6c63acc69 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 21:38:03 +0000 Subject: [PATCH 05/16] Split out Svn VCS --- CHANGELOG.rst | 2 +- dfetch/commands/common.py | 6 +- dfetch/commands/diff.py | 3 +- dfetch/commands/import_.py | 6 +- dfetch/project/svn.py | 304 ++++-------------------------------- dfetch/vcs/svn.py | 310 +++++++++++++++++++++++++++++++++++++ tests/test_import.py | 10 +- tests/test_svn.py | 44 +++--- 8 files changed, 372 insertions(+), 313 deletions(-) create mode 100644 dfetch/vcs/svn.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c34c4344..30c223f2d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Release 0.12.0 (unreleased) ==================================== -* Under development... +* Internal refactoring (#896) Release 0.11.0 (released 2026-01-03) ==================================== diff --git a/dfetch/commands/common.py b/dfetch/commands/common.py index 7a1c1557e..fcebffe47 100644 --- a/dfetch/commands/common.py +++ b/dfetch/commands/common.py @@ -8,8 +8,8 @@ from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest, get_childmanifests from dfetch.manifest.project import ProjectEntry -from dfetch.project.svn import SvnSubProject from dfetch.vcs.git import GitLocalRepo +from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -72,8 +72,8 @@ def files_to_ignore(path: str) -> Sequence[str]: """Return a list of files that can be ignored in a given path.""" if GitLocalRepo().is_git(): ignore_list = GitLocalRepo.ignored_files(path) - elif SvnSubProject.check_path(): - ignore_list = SvnSubProject.ignored_files(path) + elif SvnRepo().is_svn(): + ignore_list = SvnRepo.ignored_files(path) else: ignore_list = [] return ignore_list diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index 6fb5138e4..df6bb949a 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -107,6 +107,7 @@ from dfetch.project.svn import SvnSubProject from dfetch.util.util import catch_runtime_exceptions, in_directory from dfetch.vcs.git import GitLocalRepo +from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -173,7 +174,7 @@ def _get_repo(superproject: SuperProject, project: ProjectEntry) -> SubProject: ) if GitLocalRepo(superproject.root_directory).is_git(): return GitSubProject(project) - if SvnSubProject.check_path(superproject.root_directory): + if SvnRepo(superproject.root_directory).is_svn(): return SvnSubProject(project) raise RuntimeError( diff --git a/dfetch/commands/import_.py b/dfetch/commands/import_.py index a35dc281c..f54ccde27 100644 --- a/dfetch/commands/import_.py +++ b/dfetch/commands/import_.py @@ -92,8 +92,8 @@ from dfetch.manifest.manifest import Manifest from dfetch.manifest.project import ProjectEntry from dfetch.manifest.remote import Remote -from dfetch.project.svn import SvnSubProject from dfetch.vcs.git import GitLocalRepo +from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -139,7 +139,7 @@ def _import_projects() -> Sequence[ProjectEntry]: """Find out what type of VCS is used and import projects.""" if GitLocalRepo().is_git(): projects = _import_from_git() - elif SvnSubProject.check_path(): + elif SvnRepo().is_svn(): projects = _import_from_svn() else: raise RuntimeError( @@ -152,7 +152,7 @@ def _import_projects() -> Sequence[ProjectEntry]: def _import_from_svn() -> Sequence[ProjectEntry]: projects: list[ProjectEntry] = [] - for external in SvnSubProject.externals(): + for external in SvnRepo.externals(): projects.append( ProjectEntry( { diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 81382e049..8c730ee97 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -2,158 +2,40 @@ import os import pathlib -import re import urllib.parse from collections.abc import Sequence -from typing import NamedTuple, Optional +from typing import Optional from dfetch.log import get_logger +from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version from dfetch.project.subproject import SubProject -from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline from dfetch.util.util import ( find_matching_files, find_non_matching_files, in_directory, safe_rm, ) -from dfetch.vcs.patch import ( - combine_patches, - create_svn_patch_for_new_file, - filter_patch, -) +from dfetch.vcs.patch import combine_patches, create_svn_patch_for_new_file +from dfetch.vcs.svn import SvnRemote, SvnRepo, get_svn_version logger = get_logger(__name__) -class External(NamedTuple): - """Information about a svn external.""" - - name: str - toplevel: str - path: str - revision: str - url: str - branch: str - tag: str - src: str - - class SvnSubProject(SubProject): """A svn subproject.""" - DEFAULT_BRANCH = "trunk" NAME = "svn" - @staticmethod - def externals() -> list[External]: - """Get list of externals.""" - result = run_on_cmdline( - logger, - [ - "svn", - "--non-interactive", - "propget", - "svn:externals", - "-R", - ], - ) - - repo_root = SvnSubProject._get_info_from_target()["Repository Root"] - - externals: list[External] = [] - path_pattern = r"([^\s^-]+)\s+-" - for entry in result.stdout.decode().split(os.linesep * 2): - match: Optional[re.Match[str]] = None - local_path: str = "" - for match in re.finditer(path_pattern, entry): - pass - if match: - local_path = match.group(1) - entry = re.sub(path_pattern, "", entry) - - for match in re.finditer( - r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)", - entry, - ): - url = match.group(1) or match.group(4) - name = match.group(3) or match.group(5) - rev = "" if not match.group(2) else match.group(2).strip() - - url, branch, tag, src = SvnSubProject._split_url(url, repo_root) - - externals += [ - External( - name=name, - toplevel=os.getcwd(), - path="/".join(os.path.join(local_path, name).split(os.sep)), - revision=rev, - url=url, - branch=branch, - tag=tag, - src=src, - ) - ] - - return externals - - @staticmethod - def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: - # ../ Relative to the URL of the directory on which the svn:externals property is set - # ^/ Relative to the root of the repository in which the svn:externals property is versioned - # // Relative to the scheme of the URL of the directory on which the svn:externals property is set - # / Relative to the root URL of the server on which the svn:externals property is versioned - url = re.sub(r"^\^", repo_root, url) - branch = " " - tag = "" - src = "" - - for match in re.finditer( - r"(.*)\/(branches\/[^\/]+|tags\/[^\/]+|trunk)[\/]*(.*)", url - ): - url = match.group(1) - branch = ( - match.group(2) if match.group(2) != SvnSubProject.DEFAULT_BRANCH else "" - ) - src = match.group(3) - - path = branch.split("/") - if path[0] == "branches": - branch = path[1] - elif path[0] == "tags": - tag = path[1] - branch = "" - - if branch == " " and url.startswith(repo_root): - src = url[len(f"{repo_root}/") :] - url = repo_root - - return (url, branch, tag, src) + def __init__(self, project: ProjectEntry): + """Create a Svn subproject.""" + super().__init__(project) + self._remote_repo = SvnRemote(self.remote) + self._repo = SvnRepo(self.local_path) def check(self) -> bool: """Check if is SVN.""" - try: - run_on_cmdline(logger, ["svn", "info", self.remote, "--non-interactive"]) - return True - except SubprocessCommandError as exc: - if exc.stderr.startswith("svn: E170013"): - raise RuntimeError( - f">>>{exc.cmd}<<< failed!\n" - + f"'{self.remote}' is not a valid URL or unreachable:\n{exc.stdout or exc.stderr}" - ) from exc - return False - except RuntimeError: - return False - - @staticmethod - def check_path(path: str = ".") -> bool: - """Check if is SVN.""" - try: - with in_directory(path): - run_on_cmdline(logger, ["svn", "info", "--non-interactive"]) - return True - except (SubprocessCommandError, RuntimeError): - return False + return self._remote_repo.is_svn() @staticmethod def revision_is_enough() -> bool: @@ -162,7 +44,7 @@ def revision_is_enough() -> bool: def _latest_revision_on_branch(self, branch: str) -> str: """Get the latest revision on a branch.""" - if branch not in (self.DEFAULT_BRANCH, "", " "): + if branch not in (self._repo.DEFAULT_BRANCH, "", " "): branch = f"branches/{branch}" return self._get_revision(branch) @@ -174,28 +56,19 @@ def _does_revision_exist(self, revision: str) -> bool: def _list_of_tags(self) -> list[str]: """Get list of all available tags.""" - result = run_on_cmdline( - logger, ["svn", "ls", "--non-interactive", f"{self.remote}/tags"] - ) - return [ - str(tag).strip("/\r") for tag in result.stdout.decode().split("\n") if tag - ] + return self._remote_repo.list_of_tags() @staticmethod def list_tool_info() -> None: """Print out version information.""" try: - result = run_on_cmdline(logger, ["svn", "--version", "--non-interactive"]) + tool, version = get_svn_version() + SubProject._log_tool(tool, version) except RuntimeError as exc: logger.debug( f"Something went wrong trying to get the version of svn: {exc}" ) SubProject._log_tool("svn", "") - return - - first_line = result.stdout.decode().split("\n")[0] - tool, version = first_line.replace(",", "").split("version", maxsplit=1) - SubProject._log_tool(tool, version) def _determine_what_to_fetch(self, version: Version) -> tuple[str, str, str]: """Based on the given version, determine what to fetch. @@ -216,11 +89,11 @@ def _determine_what_to_fetch(self, version: Version) -> tuple[str, str, str]: branch_path = "" branch = " " else: - branch = version.branch or self.DEFAULT_BRANCH + branch = version.branch or self._repo.DEFAULT_BRANCH branch_path = ( f"branches/{branch}" - if branch != self.DEFAULT_BRANCH - else self.DEFAULT_BRANCH + if branch != self._repo.DEFAULT_BRANCH + else self._repo.DEFAULT_BRANCH ) branch_path = urllib.parse.quote(branch_path) @@ -254,7 +127,7 @@ def _fetch_impl(self, version: Version) -> Version: complete_path, file_pattern = self._parse_file_pattern(complete_path) - SvnSubProject._export(complete_path, rev_arg, self.local_path) + self._repo.export(complete_path, rev_arg, self.local_path) if file_pattern: for file in find_non_matching_files(self.local_path, (file_pattern,)): @@ -273,7 +146,7 @@ def _fetch_impl(self, version: Version) -> Version: if os.path.isdir(self.local_path) else os.path.dirname(self.local_path) ) - SvnSubProject._export(f"{root_branch_path}/{file}", rev_arg, dest) + self._repo.export(f"{root_branch_path}/{file}", rev_arg, dest) break if self.ignore: @@ -294,125 +167,40 @@ def _parse_file_pattern(complete_path: str) -> tuple[str, str]: return complete_path, glob_filter def _get_info(self, branch: str) -> dict[str, str]: - return self._get_info_from_target(f"{self.remote}/{branch}") - - @staticmethod - def _export(url: str, rev: str = "", dst: str = ".") -> None: - run_on_cmdline( - logger, - ["svn", "export", "--non-interactive", "--force"] - + rev.split(" ") - + [url, dst], - ) - - @staticmethod - def _files_in_path(url_path: str) -> list[str]: - return [ - str(line) - for line in run_on_cmdline( - logger, ["svn", "list", "--non-interactive", url_path] - ) - .stdout.decode() - .splitlines() - ] + return self._repo.get_info_from_target(f"{self.remote}/{branch}") @staticmethod def _license_files(url_path: str) -> list[str]: return [ str(license) for license in filter( - SvnSubProject.is_license_file, SvnSubProject._files_in_path(url_path) + SvnSubProject.is_license_file, SvnRepo.files_in_path(url_path) ) ] - @staticmethod - def _get_info_from_target(target: str = "") -> dict[str, str]: - try: - result = run_on_cmdline( - logger, ["svn", "info", "--non-interactive", target.strip()] - ).stdout.decode() - except SubprocessCommandError as exc: - if exc.stderr.startswith("svn: E170013"): - raise RuntimeError( - f">>>{exc.cmd}<<< failed!\n" - + f"'{target.strip()}' is not a valid URL or unreachable:\n{exc.stderr or exc.stdout}" - ) from exc - raise - - return { - key.strip(): value.strip() - for key, value in ( - line.split(":", maxsplit=1) for line in result.split(os.linesep) if line - ) - } - def _get_revision(self, branch: str) -> str: return self._get_info(branch)["Revision"] - @staticmethod - def _get_last_changed_revision(target: str) -> str: - if os.path.isdir(target): - last_digits = re.compile(r"(?P\d+)(?!.*\d)") - version = run_on_cmdline( - logger, ["svnversion", target.strip()] - ).stdout.decode() - - parsed_version = last_digits.search(version) - if parsed_version: - return parsed_version.group("digits") - raise RuntimeError(f"svnversion output was unexpected: {version}") - - return str( - run_on_cmdline( - logger, - [ - "svn", - "info", - "--non-interactive", - "--show-item", - "last-changed-revision", - target.strip(), - ], - ) - .stdout.decode() - .strip() - ) - def metadata_revision(self) -> str: """Get the revision of the metadata file.""" - return self._get_last_changed_revision(self.metadata_path) + return self._repo.get_last_changed_revision(self.metadata_path) def current_revision(self) -> str: """Get the current revision of the repo.""" - return self._get_last_changed_revision(self.local_path) + return self._repo.get_last_changed_revision(self.local_path) def get_diff( self, old_revision: str, new_revision: Optional[str], ignore: Sequence[str] ) -> str: """Get the diff between two revisions.""" - cmd = [ - "svn", - "diff", - "--non-interactive", - "--ignore-properties", - ".", - "-r", - old_revision, - ] - if new_revision: - cmd[-1] += f":{new_revision}" - - with in_directory(self.local_path): - patch_text = run_on_cmdline(logger, cmd).stdout - - filtered = filter_patch(patch_text, ignore) + filtered = self._repo.create_diff(old_revision, new_revision, ignore) if new_revision: return filtered patches: list[bytes] = [filtered.encode("utf-8")] if filtered else [] with in_directory(self.local_path): - for file_path in self._untracked_files(".", ignore): + for file_path in self._repo.untracked_files(".", ignore): patch = create_svn_patch_for_new_file(file_path) if patch: patches.append(patch.encode("utf-8")) @@ -421,44 +209,4 @@ def get_diff( def get_default_branch(self) -> str: """Get the default branch of this repository.""" - return self.DEFAULT_BRANCH - - @staticmethod - def _untracked_files(path: str, ignore: Sequence[str]) -> list[str]: - """Get list of untracked files in the working copy.""" - result = ( - run_on_cmdline( - logger, - ["svn", "status", "--non-interactive", path], - ) - .stdout.decode() - .splitlines() - ) - - files = [] - for line in result: - if line.startswith("?"): - file_path = line[1:].strip() - if not any( - pathlib.Path(file_path).match(pattern) for pattern in ignore - ): - files.append(file_path) - return files - - @staticmethod - def ignored_files(path: str) -> Sequence[str]: - """Get list of ignored files in the working copy.""" - if not pathlib.Path(path).exists(): - return [] - - with in_directory(path): - result = ( - run_on_cmdline( - logger, - ["svn", "status", "--non-interactive", "--no-ignore", "."], - ) - .stdout.decode() - .splitlines() - ) - - return [line[1:].strip() for line in result if line.startswith("I")] + return self._repo.DEFAULT_BRANCH diff --git a/dfetch/vcs/svn.py b/dfetch/vcs/svn.py new file mode 100644 index 000000000..e3e2b20cf --- /dev/null +++ b/dfetch/vcs/svn.py @@ -0,0 +1,310 @@ +"""Svn repository.""" + +import os +import pathlib +import re +from collections.abc import Sequence +from typing import NamedTuple, Optional + +from dfetch.log import get_logger +from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline +from dfetch.util.util import in_directory +from dfetch.vcs.patch import filter_patch + +logger = get_logger(__name__) + + +def get_svn_version() -> tuple[str, str]: + """Get the name and version of git.""" + result = run_on_cmdline(logger, ["svn", "--version", "--non-interactive"]) + first_line = result.stdout.decode().split("\n")[0] + tool, version = first_line.replace(",", "").split("version", maxsplit=1) + return (str(tool), str(version)) + + +class External(NamedTuple): + """Information about a svn external.""" + + name: str + toplevel: str + path: str + revision: str + url: str + branch: str + tag: str + src: str + + +class SvnRemote: + """A remote svn repository.""" + + def __init__(self, remote: str) -> None: + """Create a git remote repo.""" + self._remote = remote + + def is_svn(self) -> bool: + """Check if is SVN.""" + try: + run_on_cmdline(logger, ["svn", "info", self._remote, "--non-interactive"]) + return True + except SubprocessCommandError as exc: + if exc.stderr.startswith("svn: E170013"): + raise RuntimeError( + f">>>{exc.cmd}<<< failed!\n" + + f"'{self._remote}' is not a valid URL or unreachable:\n{exc.stdout or exc.stderr}" + ) from exc + return False + except RuntimeError: + return False + + def list_of_tags(self) -> list[str]: + """Get list of all available tags.""" + result = run_on_cmdline( + logger, ["svn", "ls", "--non-interactive", f"{self._remote}/tags"] + ) + return [ + str(tag).strip("/\r") for tag in result.stdout.decode().split("\n") if tag + ] + + +class SvnRepo: + """An svn repository.""" + + DEFAULT_BRANCH = "trunk" + + def __init__( + self, + path: str = ".", + ) -> None: + """Create a svn repo.""" + self._path = path + + def is_svn(self) -> bool: + """Check if is SVN.""" + try: + with in_directory(self._path): + run_on_cmdline(logger, ["svn", "info", "--non-interactive"]) + return True + except (SubprocessCommandError, RuntimeError): + return False + + @staticmethod + def externals() -> list[External]: + """Get list of externals.""" + result = run_on_cmdline( + logger, + [ + "svn", + "--non-interactive", + "propget", + "svn:externals", + "-R", + ], + ) + + repo_root = SvnRepo.get_info_from_target()["Repository Root"] + + externals: list[External] = [] + path_pattern = r"([^\s^-]+)\s+-" + for entry in result.stdout.decode().split(os.linesep * 2): + match: Optional[re.Match[str]] = None + local_path: str = "" + for match in re.finditer(path_pattern, entry): + pass + if match: + local_path = match.group(1) + entry = re.sub(path_pattern, "", entry) + + for match in re.finditer( + r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)", + entry, + ): + url = match.group(1) or match.group(4) + name = match.group(3) or match.group(5) + rev = "" if not match.group(2) else match.group(2).strip() + + url, branch, tag, src = SvnRepo._split_url(url, repo_root) + + externals += [ + External( + name=name, + toplevel=os.getcwd(), + path="/".join(os.path.join(local_path, name).split(os.sep)), + revision=rev, + url=url, + branch=branch, + tag=tag, + src=src, + ) + ] + + return externals + + @staticmethod + def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: + # ../ Relative to the URL of the directory on which the svn:externals property is set + # ^/ Relative to the root of the repository in which the svn:externals property is versioned + # // Relative to the scheme of the URL of the directory on which the svn:externals property is set + # / Relative to the root URL of the server on which the svn:externals property is versioned + url = re.sub(r"^\^", repo_root, url) + branch = " " + tag = "" + src = "" + + for match in re.finditer( + r"(.*)\/(branches\/[^\/]+|tags\/[^\/]+|trunk)[\/]*(.*)", url + ): + url = match.group(1) + branch = match.group(2) if match.group(2) != SvnRepo.DEFAULT_BRANCH else "" + src = match.group(3) + + path = branch.split("/") + if path[0] == "branches": + branch = path[1] + elif path[0] == "tags": + tag = path[1] + branch = "" + + if branch == " " and url.startswith(repo_root): + src = url[len(f"{repo_root}/") :] + url = repo_root + + return (url, branch, tag, src) + + @staticmethod + def get_info_from_target(target: str = "") -> dict[str, str]: + """Get the info of the given target.""" + try: + result = run_on_cmdline( + logger, ["svn", "info", "--non-interactive", target.strip()] + ).stdout.decode() + except SubprocessCommandError as exc: + if exc.stderr.startswith("svn: E170013"): + raise RuntimeError( + f">>>{exc.cmd}<<< failed!\n" + + f"'{target.strip()}' is not a valid URL or unreachable:\n{exc.stderr or exc.stdout}" + ) from exc + raise + + return { + key.strip(): value.strip() + for key, value in ( + line.split(":", maxsplit=1) for line in result.split(os.linesep) if line + ) + } + + @staticmethod + def get_last_changed_revision(target: str) -> str: + """Get the last changed revision of the given target.""" + if os.path.isdir(target): + last_digits = re.compile(r"(?P\d+)(?!.*\d)") + version = run_on_cmdline( + logger, ["svnversion", target.strip()] + ).stdout.decode() + + parsed_version = last_digits.search(version) + if parsed_version: + return parsed_version.group("digits") + raise RuntimeError(f"svnversion output was unexpected: {version}") + + return str( + run_on_cmdline( + logger, + [ + "svn", + "info", + "--non-interactive", + "--show-item", + "last-changed-revision", + target.strip(), + ], + ) + .stdout.decode() + .strip() + ) + + @staticmethod + def untracked_files(path: str, ignore: Sequence[str]) -> list[str]: + """Get list of untracked files in the working copy.""" + result = ( + run_on_cmdline( + logger, + ["svn", "status", "--non-interactive", path], + ) + .stdout.decode() + .splitlines() + ) + + files = [] + for line in result: + if line.startswith("?"): + file_path = line[1:].strip() + if not any( + pathlib.Path(file_path).match(pattern) for pattern in ignore + ): + files.append(file_path) + return files + + @staticmethod + def export(url: str, rev: str = "", dst: str = ".") -> None: + """Export the given revision from url to destination.""" + run_on_cmdline( + logger, + ["svn", "export", "--non-interactive", "--force"] + + rev.split(" ") + + [url, dst], + ) + + @staticmethod + def files_in_path(url_path: str) -> list[str]: + """List all files in path at the given url.""" + return [ + str(line) + for line in run_on_cmdline( + logger, ["svn", "list", "--non-interactive", url_path] + ) + .stdout.decode() + .splitlines() + ] + + @staticmethod + def ignored_files(path: str) -> Sequence[str]: + """Get list of ignored files in the working copy.""" + if not pathlib.Path(path).exists(): + return [] + + with in_directory(path): + result = ( + run_on_cmdline( + logger, + ["svn", "status", "--non-interactive", "--no-ignore", "."], + ) + .stdout.decode() + .splitlines() + ) + + return [line[1:].strip() for line in result if line.startswith("I")] + + def create_diff( + self, + old_revision: str, + new_revision: Optional[str], + ignore: Sequence[str], + ) -> str: + """Generate a relative diff patch.""" + cmd = [ + "svn", + "diff", + "--non-interactive", + "--ignore-properties", + ".", + "-r", + old_revision, + ] + if new_revision: + cmd[-1] += f":{new_revision}" + + with in_directory(self._path): + patch_text = run_on_cmdline(logger, cmd).stdout + + return filter_patch(patch_text, ignore) diff --git a/tests/test_import.py b/tests/test_import.py index 04d23818f..933e0c82f 100644 --- a/tests/test_import.py +++ b/tests/test_import.py @@ -9,8 +9,8 @@ import pytest from dfetch.commands.import_ import Import -from dfetch.project.svn import External from dfetch.vcs.git import Submodule +from dfetch.vcs.svn import External DEFAULT_ARGS = argparse.Namespace(non_recursive=False) @@ -105,14 +105,12 @@ def test_git_import(name, submodules): def test_svn_import(name, externals): import_ = Import() - with patch("dfetch.commands.import_.SvnSubProject.check_path") as check_path: - with patch( - "dfetch.commands.import_.SvnSubProject.externals" - ) as mocked_externals: + with patch("dfetch.commands.import_.SvnRepo.is_svn") as is_svn: + with patch("dfetch.commands.import_.SvnRepo.externals") as mocked_externals: with patch("dfetch.commands.import_.Manifest") as mocked_manifest: with patch("dfetch.commands.import_.GitLocalRepo.is_git") as is_git: is_git.return_value = False - check_path.return_value = True + is_svn.return_value = True mocked_externals.return_value = externals if len(externals) == 0: diff --git a/tests/test_svn.py b/tests/test_svn.py index 985d81faa..1d5b6b905 100644 --- a/tests/test_svn.py +++ b/tests/test_svn.py @@ -9,8 +9,9 @@ import pytest from dfetch.manifest.project import ProjectEntry -from dfetch.project.svn import External, SvnSubProject +from dfetch.project.svn import SvnSubProject from dfetch.util.cmdline import SubprocessCommandError +from dfetch.vcs.svn import External, SvnRemote, SvnRepo REPO_ROOT = "repo-root" CWD = "C:\\mydir" @@ -129,17 +130,15 @@ ], ) def test_externals(name, externals, expectations): - with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: - with patch( - "dfetch.project.svn.SvnSubProject._get_info_from_target" - ) as target_info_mock: - with patch("dfetch.project.svn.os.getcwd") as cwd_mock: + with patch("dfetch.vcs.svn.run_on_cmdline") as run_on_cmdline_mock: + with patch("dfetch.vcs.svn.SvnRepo.get_info_from_target") as target_info_mock: + with patch("dfetch.vcs.svn.os.getcwd") as cwd_mock: cmd_output = str(os.linesep * 2).join(externals) run_on_cmdline_mock().stdout = cmd_output.encode("utf-8") target_info_mock.return_value = {"Repository Root": REPO_ROOT} cwd_mock.return_value = CWD - parsed_externals = SvnSubProject.externals() + parsed_externals = SvnRepo.externals() for actual, expected in zip( parsed_externals, expectations # , strict=True @@ -156,35 +155,33 @@ def test_externals(name, externals, expectations): ], ) def test_check_path(name, cmd_result, expectation): - with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: + with patch("dfetch.vcs.svn.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result - assert SvnSubProject.check_path() == expectation + assert SvnRepo().is_svn() == expectation @pytest.mark.parametrize( - "name, project, cmd_result, expectation", + "name, cmd_result, expectation", [ - ("Ok url", ProjectEntry({"name": "proj1", "url": "some_url"}), ["Yep!"], True), + ("Ok url", ["Yep!"], True), ( "Failed command", - ProjectEntry({"name": "proj2", "url": "some_url"}), [SubprocessCommandError], False, ), ( "No svn", - ProjectEntry({"name": "proj3", "url": "some_url"}), [RuntimeError], False, ), ], ) -def test_check(name, project, cmd_result, expectation): - with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: +def test_check(name, cmd_result, expectation): + with patch("dfetch.vcs.svn.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result - assert SvnSubProject(project).check() == expectation + assert SvnRemote("some_url").is_svn() == expectation SVN_INFO = """ @@ -203,11 +200,11 @@ def test_check(name, project, cmd_result, expectation): def test_get_info(): - with patch("dfetch.project.svn.run_on_cmdline") as run_on_cmdline_mock: + with patch("dfetch.vcs.svn.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.return_value.stdout = os.linesep.join( SVN_INFO.split("\n") ).encode() - result = SvnSubProject._get_info_from_target("bla") + result = SvnRepo.get_info_from_target("bla") expectation = { "Path": "cpputest", @@ -226,12 +223,17 @@ def test_get_info(): @pytest.fixture -def svn_repo(): +def svn_subproject(): return SvnSubProject(ProjectEntry({"name": "proj3", "url": "some_url"})) -def test_svn_repo_name(svn_repo): - assert svn_repo.NAME == "svn" +def test_svn_repo_name(svn_subproject): + assert svn_subproject.NAME == "svn" + + +@pytest.fixture +def svn_repo(): + return SvnRepo() def test_svn_repo_default_branch(svn_repo): From bee41069a4896de8afb0340b111dd83007d43c9c Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 21:40:14 +0000 Subject: [PATCH 06/16] don't use vcs name --- dfetch/commands/environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dfetch/commands/environment.py b/dfetch/commands/environment.py index da37f434a..99109baf7 100644 --- a/dfetch/commands/environment.py +++ b/dfetch/commands/environment.py @@ -24,5 +24,5 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, _: argparse.Namespace) -> None: """Perform listing the environment.""" logger.print_info_line("platform", f"{platform.system()} {platform.release()}") - for vcs in SUPPORTED_PROJECT_TYPES: - vcs.list_tool_info() + for project_type in SUPPORTED_PROJECT_TYPES: + project_type.list_tool_info() From a12a61cce683e5557a5021b42c3af287261cc54e Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 21:49:13 +0000 Subject: [PATCH 07/16] Minor typo fix --- dfetch/vcs/svn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dfetch/vcs/svn.py b/dfetch/vcs/svn.py index e3e2b20cf..bed92fe4e 100644 --- a/dfetch/vcs/svn.py +++ b/dfetch/vcs/svn.py @@ -15,7 +15,7 @@ def get_svn_version() -> tuple[str, str]: - """Get the name and version of git.""" + """Get the name and version of svn.""" result = run_on_cmdline(logger, ["svn", "--version", "--non-interactive"]) first_line = result.stdout.decode().split("\n")[0] tool, version = first_line.replace(",", "").split("version", maxsplit=1) @@ -39,7 +39,7 @@ class SvnRemote: """A remote svn repository.""" def __init__(self, remote: str) -> None: - """Create a git remote repo.""" + """Create a svn remote repo.""" self._remote = remote def is_svn(self) -> bool: From c574c352cd70f4f802ba48abc6d0c3344944a303 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 22:00:17 +0000 Subject: [PATCH 08/16] Describe the sub & super projects concepts in the getting started --- doc/getting_started.rst | 42 ++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/doc/getting_started.rst b/doc/getting_started.rst index 5620abe67..6d789e840 100644 --- a/doc/getting_started.rst +++ b/doc/getting_started.rst @@ -5,22 +5,34 @@ Getting Started Main concepts ------------- -Your project depends on other projects to build or run. In order keep the dependencies -with your project. *Dfetch* can fetch these dependencies and place them with your code. +Your project depends on other projects to build or run. In order to keep these +dependencies together with your own code, *Dfetch* can fetch these and place them +inside your project. For clarity lets call your project the *superproject* and the +dependencies the *sub-projects*. + +*Dfetch* operates on a *superproject*: the project that contains the +:ref:`Manifest` file. The superproject depends on one or more *subprojects*, +which are the external projects listed in the manifest. The superproject itself +can use any version control system (or even none at all). + +The :ref:`Manifest` file describes all the :ref:`Projects` the superproject +depends on. These subprojects can be a mix of different version control systems, +such as Git or Subversion. You can then let *Dfetch* :ref:`update` your +dependencies based on this manifest through ``git`` or ``svn``. + +*Dfetch* will fetch the correct version of each subproject and place it in the +location of your choice. If the destination folder already exists and the +version has changed, *Dfetch* will overwrite the folder with the updated +contents. + +Since any version control information (such as ``.git`` or ``.svn`` directories) is +removed from the fetched subprojects, *Dfetch* stores general information about each +subproject in a ``.dfetch_data.yaml`` file inside the fetched directory, referred to +as the metadata. + +After updating, you can then review the changes using the version control system of +your superproject and commit them as you see fit. -*Dfetch* starts from a :ref:`Manifest` file. This contains all the :ref:`Projects` -this project is depending on. You as a user can then let *Dfetch* :ref:`update` -your dependencies. - -*Dfetch* will fetch the correct version of your dependencies and place them in the -location of your choice. If the folder already exists and the version was updated -*Dfetch* will overwrite the folder with the changes. - -Since the version control information (`.git` or `svn`) is removed, *Dfetch* stores some -general information about the project in a `.dfetch_data.yaml` file inside the fetched project. - -You can then review the changes in your favorite version control system and commit -the changes as you please. My first manifest ----------------- From 6b8a7262852cb2d4d0d45ad6ebe7b8fee23bd50f Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 4 Jan 2026 22:16:54 +0000 Subject: [PATCH 09/16] Move diff into subproject --- dfetch/commands/diff.py | 38 +++++++----------------------------- dfetch/project/git.py | 2 +- dfetch/project/subproject.py | 25 ++++++++++++++++++++++-- dfetch/project/svn.py | 2 +- tests/test_subproject.py | 2 +- 5 files changed, 33 insertions(+), 36 deletions(-) diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index df6bb949a..9559fdde0 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -95,13 +95,13 @@ import argparse import os +import pathlib import dfetch.commands.command import dfetch.project from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.project.git import GitSubProject -from dfetch.project.metadata import Metadata from dfetch.project.subproject import SubProject from dfetch.project.superproject import SuperProject from dfetch.project.svn import SvnSubProject @@ -155,19 +155,18 @@ def __call__(self, args: argparse.Namespace) -> None: for project in projects: patch_name = f"{project.name}.patch" with catch_runtime_exceptions(exceptions) as exceptions: - repo = _get_repo(superproject, project) - patch = _diff_from_repo(repo, project, revs) + patch = _get_sub_project(superproject, project).diff(revs) _dump_patch( - superproject.manifest.path, revs, project, patch_name, patch + superproject.root_directory, revs, project, patch_name, patch ) if exceptions: raise RuntimeError("\n".join(exceptions)) -def _get_repo(superproject: SuperProject, project: ProjectEntry) -> SubProject: - """Get the repo type from the project.""" +def _get_sub_project(superproject: SuperProject, project: ProjectEntry) -> SubProject: + """Get the subproject in the same vcs type as the superproject.""" if not os.path.exists(project.destination): raise RuntimeError( "You cannot generate a diff of a project that was never fetched" @@ -178,29 +177,7 @@ def _get_repo(superproject: SuperProject, project: ProjectEntry) -> SubProject: return SvnSubProject(project) raise RuntimeError( - "Can only create patch in SVN or Git repo", - ) - - -def _diff_from_repo(repo: SubProject, project: ProjectEntry, revs: list[str]) -> str: - """Generate a relative diff for a svn repo.""" - if len(revs) > 2: - raise RuntimeError(f"Too many revisions given! {revs}") - - if not revs: - revs.append(repo.metadata_revision()) - if not revs[-1]: - raise RuntimeError( - "When not providing any commits, dfetch starts from" - f" the last commit to {Metadata.FILENAME} in {project.destination}." - " Please either commit this, or specify a revision to start from with --revs" - ) - - if len(revs) == 1: - revs.append("") - - return repo.get_diff( - old_revision=revs[0], new_revision=revs[1], ignore=(Metadata.FILENAME,) + "Can only create patch if your project is an SVN or Git repo", ) @@ -214,8 +191,7 @@ def _dump_patch( project.name, f"Generating patch {patch_name} {rev_range} in {os.path.dirname(path)}", ) - with open(patch_name, "w", encoding="UTF-8") as patch_file: - patch_file.write(patch) + pathlib.Path(patch_name).write_text(patch, encoding="UTF-8") else: if revs[1]: msg = f"No diffs found from {revs[0]} to {revs[1]}" diff --git a/dfetch/project/git.py b/dfetch/project/git.py index 9542ce418..45edd6e62 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -51,7 +51,7 @@ def current_revision(self) -> str: """Get the revision of the metadata file.""" return str(self._local_repo.get_current_hash()) - def get_diff( + def _diff_impl( self, old_revision: str, new_revision: Optional[str], ignore: Sequence[str] ) -> str: """Get the diff of two revisions.""" diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index e52de9e94..e6d9109f1 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -373,13 +373,13 @@ def current_revision(self) -> str: """Get the revision of the metadata file.""" @abstractmethod - def get_diff( + def _diff_impl( self, old_revision: str, # noqa new_revision: Optional[str], # noqa ignore: Sequence[str], ) -> str: - """Get the diff of two revisions.""" + """Get the diff of two revisions, should be implemented by the child class.""" @abstractmethod def get_default_branch(self) -> str: @@ -392,3 +392,24 @@ def is_license_file(filename: str) -> bool: fnmatch.fnmatch(filename.lower(), pattern) for pattern in SubProject.LICENSE_GLOBS ) + + def diff(self, revs: list[str]) -> str: + """Generate a relative diff for a subproject.""" + if len(revs) > 2: + raise RuntimeError(f"Too many revisions given! {revs}") + + if not revs: + revs.append(self.metadata_revision()) + if not revs[-1]: + raise RuntimeError( + "When not providing any revisions, dfetch starts from" + f" the last revision to {Metadata.FILENAME} in {self.local_path}." + " Please either revision this, or specify a revision to start from with --revs" + ) + + if len(revs) == 1: + revs.append("") + + return self._diff_impl( + old_revision=revs[0], new_revision=revs[1], ignore=(Metadata.FILENAME,) + ) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 8c730ee97..ea02811f8 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -189,7 +189,7 @@ def current_revision(self) -> str: """Get the current revision of the repo.""" return self._repo.get_last_changed_revision(self.local_path) - def get_diff( + def _diff_impl( self, old_revision: str, new_revision: Optional[str], ignore: Sequence[str] ) -> str: """Get the diff between two revisions.""" diff --git a/tests/test_subproject.py b/tests/test_subproject.py index ce0079be3..b8137710c 100644 --- a/tests/test_subproject.py +++ b/tests/test_subproject.py @@ -49,7 +49,7 @@ def current_revision(self): def metadata_revision(self): return "1" - def get_diff(self, old_revision, new_revision, ignore): + def _diff_impl(self, old_revision, new_revision, ignore): return "" def get_default_branch(self): From 101c330e3aa3f079442f6becc1c6e70f9edfd743 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Mon, 5 Jan 2026 07:56:10 +0000 Subject: [PATCH 10/16] Simplify diff --- dfetch/commands/diff.py | 84 +++++++++++++++------------------- dfetch/project/subproject.py | 26 ++++------- dfetch/project/superproject.py | 15 ++++++ 3 files changed, 59 insertions(+), 66 deletions(-) diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index 9559fdde0..c70e77135 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -98,16 +98,9 @@ import pathlib import dfetch.commands.command -import dfetch.project from dfetch.log import get_logger -from dfetch.manifest.project import ProjectEntry -from dfetch.project.git import GitSubProject -from dfetch.project.subproject import SubProject from dfetch.project.superproject import SuperProject -from dfetch.project.svn import SvnSubProject from dfetch.util.util import catch_runtime_exceptions, in_directory -from dfetch.vcs.git import GitLocalRepo -from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -143,7 +136,7 @@ def create_menu(subparsers: dfetch.commands.command.SubparserActionType) -> None def __call__(self, args: argparse.Namespace) -> None: """Perform the diff.""" superproject = SuperProject() - revs = [r for r in args.revs.strip(":").split(":", maxsplit=1) if r] + old_rev, new_rev = self._parse_revs(args.revs) with in_directory(superproject.root_directory): exceptions: list[str] = [] @@ -153,49 +146,44 @@ def __call__(self, args: argparse.Namespace) -> None: f"No (such) project found! {', '.join(args.projects)}" ) for project in projects: - patch_name = f"{project.name}.patch" with catch_runtime_exceptions(exceptions) as exceptions: - patch = _get_sub_project(superproject, project).diff(revs) - - _dump_patch( - superproject.root_directory, revs, project, patch_name, patch - ) + if not os.path.exists(project.destination): + raise RuntimeError( + "You cannot generate a diff of a project that was never fetched" + ) + subproject = superproject.get_sub_project(project) + + if subproject is None: + raise RuntimeError( + "Can only create patch if your project is an SVN or Git repo", + ) + old_rev = old_rev or subproject.metadata_revision() + patch = subproject.diff(old_rev, new_rev) + + msg = self._rev_msg(old_rev, new_rev) + if patch: + patch_path = pathlib.Path(f"{project.name}.patch") + logger.print_info_line( + project.name, + f"Generating patch {patch_path} {msg} in {superproject.root_directory}", + ) + patch_path.write_text(patch, encoding="UTF-8") + else: + logger.print_info_line(project.name, f"No diffs found {msg}") if exceptions: raise RuntimeError("\n".join(exceptions)) + @staticmethod + def _parse_revs(revs_arg: str) -> tuple[str, str]: + revs = [r for r in revs_arg.strip(":").split(":", maxsplit=1) if r] -def _get_sub_project(superproject: SuperProject, project: ProjectEntry) -> SubProject: - """Get the subproject in the same vcs type as the superproject.""" - if not os.path.exists(project.destination): - raise RuntimeError( - "You cannot generate a diff of a project that was never fetched" - ) - if GitLocalRepo(superproject.root_directory).is_git(): - return GitSubProject(project) - if SvnRepo(superproject.root_directory).is_svn(): - return SvnSubProject(project) - - raise RuntimeError( - "Can only create patch if your project is an SVN or Git repo", - ) - - -def _dump_patch( - path: str, revs: list[str], project: ProjectEntry, patch_name: str, patch: str -) -> None: - """Dump the patch to a file.""" - if patch: - rev_range = f"from {revs[0]} to {revs[1]}" if revs[1] else f"since {revs[0]}" - logger.print_info_line( - project.name, - f"Generating patch {patch_name} {rev_range} in {os.path.dirname(path)}", - ) - pathlib.Path(patch_name).write_text(patch, encoding="UTF-8") - else: - if revs[1]: - msg = f"No diffs found from {revs[0]} to {revs[1]}" - else: - msg = f"No diffs found since {revs[0]}" - - logger.print_info_line(project.name, msg) + if len(revs) == 0: + return "", "" + if len(revs) == 1: + return revs[0], "" + return revs[0], revs[1] + + @staticmethod + def _rev_msg(old_rev: str, new_rev: str) -> str: + return f"from {old_rev} to {new_rev}" if new_rev else f"since {old_rev}" diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index e6d9109f1..130898f92 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -393,23 +393,13 @@ def is_license_file(filename: str) -> bool: for pattern in SubProject.LICENSE_GLOBS ) - def diff(self, revs: list[str]) -> str: + def diff(self, old_rev: str, new_rev: str) -> str: """Generate a relative diff for a subproject.""" - if len(revs) > 2: - raise RuntimeError(f"Too many revisions given! {revs}") - - if not revs: - revs.append(self.metadata_revision()) - if not revs[-1]: - raise RuntimeError( - "When not providing any revisions, dfetch starts from" - f" the last revision to {Metadata.FILENAME} in {self.local_path}." - " Please either revision this, or specify a revision to start from with --revs" - ) - - if len(revs) == 1: - revs.append("") + if not old_rev: + raise RuntimeError( + "When not providing any revisions, dfetch starts from" + f" the last revision to {Metadata.FILENAME} in {self.local_path}." + " Please either revision this, or specify a revision to start from with --revs" + ) - return self._diff_impl( - old_revision=revs[0], new_revision=revs[1], ignore=(Metadata.FILENAME,) - ) + return self._diff_impl(old_rev, new_rev, ignore=(Metadata.FILENAME,)) diff --git a/dfetch/project/superproject.py b/dfetch/project/superproject.py index 4b9f59fc7..972cf75b7 100644 --- a/dfetch/project/superproject.py +++ b/dfetch/project/superproject.py @@ -12,7 +12,13 @@ from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest, find_manifest +from dfetch.manifest.project import ProjectEntry from dfetch.manifest.validate import validate +from dfetch.project.git import GitSubProject +from dfetch.project.subproject import SubProject +from dfetch.project.svn import SvnSubProject +from dfetch.vcs.git import GitLocalRepo +from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -44,3 +50,12 @@ def root_directory(self) -> str: def manifest(self) -> Manifest: """The manifest of the super project.""" return self._manifest + + def get_sub_project(self, project: ProjectEntry) -> SubProject | None: + """Get the subproject in the same vcs type as the superproject.""" + if GitLocalRepo(self.root_directory).is_git(): + return GitSubProject(project) + if SvnRepo(self.root_directory).is_svn(): + return SvnSubProject(project) + + return None From a208accf120e460eef2d0072f4053d76cd0eb1af Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 5 Jan 2026 12:09:42 +0000 Subject: [PATCH 11/16] move ignored files to superproject --- dfetch/commands/check.py | 5 +++-- dfetch/commands/common.py | 14 -------------- dfetch/commands/update.py | 4 ++-- dfetch/project/superproject.py | 17 +++++++++++++++++ 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/dfetch/commands/check.py b/dfetch/commands/check.py index 05794e0f3..156fddefd 100644 --- a/dfetch/commands/check.py +++ b/dfetch/commands/check.py @@ -32,7 +32,7 @@ import dfetch.commands.command import dfetch.project -from dfetch.commands.common import check_child_manifests, files_to_ignore +from dfetch.commands.common import check_child_manifests from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest from dfetch.project.superproject import SuperProject @@ -98,7 +98,8 @@ def __call__(self, args: argparse.Namespace) -> None: for project in superproject.manifest.selected_projects(args.projects): with catch_runtime_exceptions(exceptions) as exceptions: dfetch.project.make(project).check_for_update( - reporters, files_to_ignore=files_to_ignore(project.destination) + reporters, + files_to_ignore=superproject.ignored_files(project.destination), ) if not args.no_recommendations and os.path.isdir(project.destination): diff --git a/dfetch/commands/common.py b/dfetch/commands/common.py index fcebffe47..d73f88a8c 100644 --- a/dfetch/commands/common.py +++ b/dfetch/commands/common.py @@ -1,15 +1,12 @@ """Module for common command operations.""" import os -from collections.abc import Sequence import yaml from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest, get_childmanifests from dfetch.manifest.project import ProjectEntry -from dfetch.vcs.git import GitLocalRepo -from dfetch.vcs.svn import SvnRepo logger = get_logger(__name__) @@ -66,14 +63,3 @@ def _make_recommendation( for line in recommendation_json.splitlines(): logger.warning(line) logger.warning("") - - -def files_to_ignore(path: str) -> Sequence[str]: - """Return a list of files that can be ignored in a given path.""" - if GitLocalRepo().is_git(): - ignore_list = GitLocalRepo.ignored_files(path) - elif SvnRepo().is_svn(): - ignore_list = SvnRepo.ignored_files(path) - else: - ignore_list = [] - return ignore_list diff --git a/dfetch/commands/update.py b/dfetch/commands/update.py index 22cf5c85d..5f5656f8f 100644 --- a/dfetch/commands/update.py +++ b/dfetch/commands/update.py @@ -37,7 +37,7 @@ import dfetch.manifest.validate import dfetch.project.git import dfetch.project.svn -from dfetch.commands.common import check_child_manifests, files_to_ignore +from dfetch.commands.common import check_child_manifests from dfetch.log import get_logger from dfetch.project.superproject import SuperProject from dfetch.util.util import catch_runtime_exceptions, in_directory @@ -90,7 +90,7 @@ def __call__(self, args: argparse.Namespace) -> None: self._check_destination(project, destinations) dfetch.project.make(project).update( force=args.force, - files_to_ignore=files_to_ignore(project.destination), + files_to_ignore=superproject.ignored_files(project.destination), ) if not args.no_recommendations and os.path.isdir( diff --git a/dfetch/project/superproject.py b/dfetch/project/superproject.py index 972cf75b7..53eaab22c 100644 --- a/dfetch/project/superproject.py +++ b/dfetch/project/superproject.py @@ -9,6 +9,8 @@ from __future__ import annotations import os +import pathlib +from collections.abc import Sequence from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest, find_manifest @@ -59,3 +61,18 @@ def get_sub_project(self, project: ProjectEntry) -> SubProject | None: return SvnSubProject(project) return None + + def ignored_files(self, path: str) -> Sequence[str]: + """Return a list of files that can be ignored in a given path.""" + if ( + os.path.commonprefix((pathlib.Path(path).resolve(), self.root_directory)) + != self.root_directory + ): + raise RuntimeError(f"{path} not in superproject {self.root_directory}!") + + if GitLocalRepo(self.root_directory).is_git(): + return GitLocalRepo.ignored_files(path) + if SvnRepo(self.root_directory).is_svn(): + return SvnRepo.ignored_files(path) + + return [] From 31f20def99b8e5ff2202157d4e676ae5cb452613 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 5 Jan 2026 21:13:31 +0000 Subject: [PATCH 12/16] Review comments --- dfetch/project/subproject.py | 2 +- dfetch/vcs/svn.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 130898f92..9d7d75588 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -399,7 +399,7 @@ def diff(self, old_rev: str, new_rev: str) -> str: raise RuntimeError( "When not providing any revisions, dfetch starts from" f" the last revision to {Metadata.FILENAME} in {self.local_path}." - " Please either revision this, or specify a revision to start from with --revs" + " Please either commit this, or specify a revision to start from with --revs" ) return self._diff_impl(old_rev, new_rev, ignore=(Metadata.FILENAME,)) diff --git a/dfetch/vcs/svn.py b/dfetch/vcs/svn.py index bed92fe4e..e27b745fd 100644 --- a/dfetch/vcs/svn.py +++ b/dfetch/vcs/svn.py @@ -18,6 +18,8 @@ def get_svn_version() -> tuple[str, str]: """Get the name and version of svn.""" result = run_on_cmdline(logger, ["svn", "--version", "--non-interactive"]) first_line = result.stdout.decode().split("\n")[0] + if "version" not in first_line.lower(): + raise RuntimeError(f"Unexpected svn --version output format: {first_line}") tool, version = first_line.replace(",", "").split("version", maxsplit=1) return (str(tool), str(version)) @@ -251,7 +253,7 @@ def export(url: str, rev: str = "", dst: str = ".") -> None: run_on_cmdline( logger, ["svn", "export", "--non-interactive", "--force"] - + rev.split(" ") + + (rev.split(" ") if rev else []) + [url, dst], ) From 3735aaa80a8a3a9af506a22e1906f806250248c7 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 5 Jan 2026 21:23:27 +0000 Subject: [PATCH 13/16] Fix test --- tests/test_update.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_update.py b/tests/test_update.py index 7f61c985e..53b0fabf7 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -54,6 +54,7 @@ def test_forced_update(): fake_superproject = Mock() fake_superproject.manifest = mock_manifest([{"name": "some_project"}]) fake_superproject.root_directory = "/tmp" + fake_superproject.ignored_files.return_value = [] with patch("dfetch.commands.update.SuperProject", return_value=fake_superproject): with patch( From 7911a15457854c44a6cf9ba25c935c0d020c5f08 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 5 Jan 2026 21:27:48 +0000 Subject: [PATCH 14/16] Review comments --- dfetch/project/svn.py | 10 +++++----- dfetch/vcs/svn.py | 8 +++++++- tests/test_svn.py | 2 +- tests/test_update.py | 7 +++++-- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index ea02811f8..6b180cb24 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -127,7 +127,7 @@ def _fetch_impl(self, version: Version) -> Version: complete_path, file_pattern = self._parse_file_pattern(complete_path) - self._repo.export(complete_path, rev_arg, self.local_path) + SvnRepo.export(complete_path, rev_arg, self.local_path) if file_pattern: for file in find_non_matching_files(self.local_path, (file_pattern,)): @@ -146,7 +146,7 @@ def _fetch_impl(self, version: Version) -> Version: if os.path.isdir(self.local_path) else os.path.dirname(self.local_path) ) - self._repo.export(f"{root_branch_path}/{file}", rev_arg, dest) + SvnRepo.export(f"{root_branch_path}/{file}", rev_arg, dest) break if self.ignore: @@ -167,7 +167,7 @@ def _parse_file_pattern(complete_path: str) -> tuple[str, str]: return complete_path, glob_filter def _get_info(self, branch: str) -> dict[str, str]: - return self._repo.get_info_from_target(f"{self.remote}/{branch}") + return SvnRepo.get_info_from_target(f"{self.remote}/{branch}") @staticmethod def _license_files(url_path: str) -> list[str]: @@ -183,11 +183,11 @@ def _get_revision(self, branch: str) -> str: def metadata_revision(self) -> str: """Get the revision of the metadata file.""" - return self._repo.get_last_changed_revision(self.metadata_path) + return SvnRepo.get_last_changed_revision(self.metadata_path) def current_revision(self) -> str: """Get the current revision of the repo.""" - return self._repo.get_last_changed_revision(self.local_path) + return SvnRepo.get_last_changed_revision(self.local_path) def _diff_impl( self, old_revision: str, new_revision: Optional[str], ignore: Sequence[str] diff --git a/dfetch/vcs/svn.py b/dfetch/vcs/svn.py index e27b745fd..b8f6faf80 100644 --- a/dfetch/vcs/svn.py +++ b/dfetch/vcs/svn.py @@ -107,6 +107,7 @@ def externals() -> list[External]: repo_root = SvnRepo.get_info_from_target()["Repository Root"] externals: list[External] = [] + # Pattern matches: "path - ..." where path is the local directory path_pattern = r"([^\s^-]+)\s+-" for entry in result.stdout.decode().split(os.linesep * 2): match: Optional[re.Match[str]] = None @@ -117,6 +118,9 @@ def externals() -> list[External]: local_path = match.group(1) entry = re.sub(path_pattern, "", entry) + # Pattern matches either: + # - url@revision name (pinned) + # - url name (unpinned) for match in re.finditer( r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)", entry, @@ -191,7 +195,9 @@ def get_info_from_target(target: str = "") -> dict[str, str]: return { key.strip(): value.strip() for key, value in ( - line.split(":", maxsplit=1) for line in result.split(os.linesep) if line + line.split(":", maxsplit=1) + for line in result.split(os.linesep) + if line and ":" in line ) } diff --git a/tests/test_svn.py b/tests/test_svn.py index 1d5b6b905..53652e9a2 100644 --- a/tests/test_svn.py +++ b/tests/test_svn.py @@ -227,7 +227,7 @@ def svn_subproject(): return SvnSubProject(ProjectEntry({"name": "proj3", "url": "some_url"})) -def test_svn_repo_name(svn_subproject): +def test_svn_subproject_name(svn_subproject): assert svn_subproject.NAME == "svn" diff --git a/tests/test_update.py b/tests/test_update.py index 53b0fabf7..47dd58fee 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -66,8 +66,11 @@ def test_forced_update(): with patch("dfetch.commands.update.Update._check_destination"): mocked_get_childmanifests.return_value = [] - args = DEFAULT_ARGS - args.force = True + args = argparse.Namespace( + no_recommendations=False, + force=True, + projects=[], + ) update(args) mocked_make.return_value.update.assert_called_once_with( From a88f3f4385df818419b97767b4e972fdc31df2f4 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Tue, 6 Jan 2026 16:24:25 +0000 Subject: [PATCH 15/16] Make svn import non-static --- dfetch/commands/import_.py | 2 +- dfetch/vcs/svn.py | 106 ++++++++++++++++++------------------- tests/test_svn.py | 5 +- 3 files changed, 56 insertions(+), 57 deletions(-) diff --git a/dfetch/commands/import_.py b/dfetch/commands/import_.py index f54ccde27..4f12eb3b1 100644 --- a/dfetch/commands/import_.py +++ b/dfetch/commands/import_.py @@ -152,7 +152,7 @@ def _import_projects() -> Sequence[ProjectEntry]: def _import_from_svn() -> Sequence[ProjectEntry]: projects: list[ProjectEntry] = [] - for external in SvnRepo.externals(): + for external in SvnRepo(os.getcwd()).externals(): projects.append( ProjectEntry( { diff --git a/dfetch/vcs/svn.py b/dfetch/vcs/svn.py index b8f6faf80..1f3396c0a 100644 --- a/dfetch/vcs/svn.py +++ b/dfetch/vcs/svn.py @@ -90,61 +90,61 @@ def is_svn(self) -> bool: except (SubprocessCommandError, RuntimeError): return False - @staticmethod - def externals() -> list[External]: + def externals(self) -> list[External]: """Get list of externals.""" - result = run_on_cmdline( - logger, - [ - "svn", - "--non-interactive", - "propget", - "svn:externals", - "-R", - ], - ) + with in_directory(self._path): + result = run_on_cmdline( + logger, + [ + "svn", + "--non-interactive", + "propget", + "svn:externals", + "-R", + ], + ) - repo_root = SvnRepo.get_info_from_target()["Repository Root"] - - externals: list[External] = [] - # Pattern matches: "path - ..." where path is the local directory - path_pattern = r"([^\s^-]+)\s+-" - for entry in result.stdout.decode().split(os.linesep * 2): - match: Optional[re.Match[str]] = None - local_path: str = "" - for match in re.finditer(path_pattern, entry): - pass - if match: - local_path = match.group(1) - entry = re.sub(path_pattern, "", entry) - - # Pattern matches either: - # - url@revision name (pinned) - # - url name (unpinned) - for match in re.finditer( - r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)", - entry, - ): - url = match.group(1) or match.group(4) - name = match.group(3) or match.group(5) - rev = "" if not match.group(2) else match.group(2).strip() - - url, branch, tag, src = SvnRepo._split_url(url, repo_root) - - externals += [ - External( - name=name, - toplevel=os.getcwd(), - path="/".join(os.path.join(local_path, name).split(os.sep)), - revision=rev, - url=url, - branch=branch, - tag=tag, - src=src, - ) - ] - - return externals + repo_root = SvnRepo.get_info_from_target()["Repository Root"] + + externals: list[External] = [] + # Pattern matches: "path - ..." where path is the local directory + path_pattern = r"([^\s^-]+)\s+-" + for entry in result.stdout.decode().split(os.linesep * 2): + match: Optional[re.Match[str]] = None + local_path: str = "" + for match in re.finditer(path_pattern, entry): + pass + if match: + local_path = match.group(1) + entry = re.sub(path_pattern, "", entry) + + # Pattern matches either: + # - url@revision name (pinned) + # - url name (unpinned) + for match in re.finditer( + r"([^-\s\d][^\s]+)(?:@)(\d+)\s+([^\s]+)|([^-\s\d][^\s]+)\s+([^\s]+)", + entry, + ): + url = match.group(1) or match.group(4) + name = match.group(3) or match.group(5) + rev = "" if not match.group(2) else match.group(2).strip() + + url, branch, tag, src = SvnRepo._split_url(url, repo_root) + + externals += [ + External( + name=name, + toplevel=self._path, + path="/".join(os.path.join(local_path, name).split(os.sep)), + revision=rev, + url=url, + branch=branch, + tag=tag, + src=src, + ) + ] + + return externals @staticmethod def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: diff --git a/tests/test_svn.py b/tests/test_svn.py index 53652e9a2..42fac0a2e 100644 --- a/tests/test_svn.py +++ b/tests/test_svn.py @@ -132,13 +132,12 @@ def test_externals(name, externals, expectations): with patch("dfetch.vcs.svn.run_on_cmdline") as run_on_cmdline_mock: with patch("dfetch.vcs.svn.SvnRepo.get_info_from_target") as target_info_mock: - with patch("dfetch.vcs.svn.os.getcwd") as cwd_mock: + with patch("dfetch.vcs.svn.os.chdir"): cmd_output = str(os.linesep * 2).join(externals) run_on_cmdline_mock().stdout = cmd_output.encode("utf-8") target_info_mock.return_value = {"Repository Root": REPO_ROOT} - cwd_mock.return_value = CWD - parsed_externals = SvnRepo.externals() + parsed_externals = SvnRepo(CWD).externals() for actual, expected in zip( parsed_externals, expectations # , strict=True From c88d01199cbd7ac49398759239123e5ea78b37f4 Mon Sep 17 00:00:00 2001 From: Ben Spoor <37540691+ben-edna@users.noreply.github.com> Date: Tue, 6 Jan 2026 16:38:57 +0000 Subject: [PATCH 16/16] Update changelog --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 30c223f2d..e79d95eeb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,7 @@ Release 0.12.0 (unreleased) ==================================== -* Internal refactoring (#896) +* Internal refactoring: introduce superproject & subproject (#896) Release 0.11.0 (released 2026-01-03) ====================================