From 5a82809630a665219cba15a85d57d681b05e7087 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Mon, 12 May 2025 16:58:16 -0700 Subject: [PATCH 01/14] add validation for non remote dependencies --- devservices/configs/service_config.py | 12 ++++ devservices/utils/services.py | 8 +-- tests/configs/test_service_config.py | 100 +++++++++++++++++++++++--- 3 files changed, 103 insertions(+), 17 deletions(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index 6b1fcf34..62e459b6 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -84,6 +84,8 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: ) service_config_data = config.get("x-sentry-service-config") + docker_compose_services = config.get("services", {}).keys() + valid_dependency_keys = {field.name for field in fields(Dependency)} dependencies = {} @@ -106,6 +108,16 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: f"Error parsing service dependencies: {type_error}" ) from type_error + # Validate that all non-remote dependencies are defined in docker-compose services + for dependency_name, dependency in dependencies.items(): + if ( + dependency.remote is None + and dependency_name not in docker_compose_services + ): + raise ConfigValidationError( + f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services" + ) + service_config = ServiceConfig( version=service_config_data.get("version"), service_name=service_config_data.get("service_name"), diff --git a/devservices/utils/services.py b/devservices/utils/services.py index f8bb72a3..005bbd2b 100644 --- a/devservices/utils/services.py +++ b/devservices/utils/services.py @@ -8,7 +8,6 @@ from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError from devservices.exceptions import ServiceNotFoundError -from devservices.utils.console import Console from devservices.utils.devenv import get_coderoot @@ -23,16 +22,13 @@ def get_local_services(coderoot: str) -> list[Service]: """Get a list of services in the coderoot.""" from devservices.configs.service_config import load_service_config_from_file - console = Console() - services = [] for repo in os.listdir(coderoot): repo_path = os.path.join(coderoot, repo) try: service_config = load_service_config_from_file(repo_path) - except (ConfigParseError, ConfigValidationError) as e: - console.warning(f"{repo} was found with an invalid config: {e}") - continue + except (ConfigParseError, ConfigValidationError): + raise except ConfigNotFoundError: # Ignore repos that don't have devservices configs continue diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index bcc02933..e6797630 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -70,7 +70,13 @@ def test_load_service_config_from_file( "service_name": service_name, "dependencies": {key: value for key, value in dependencies.items()}, "modes": {key: value for key, value in modes.items()}, - } + }, + "services": { + key: { + "image": key, + } + for key in dependencies.keys() + }, } create_config_file(tmp_path, config) @@ -95,7 +101,8 @@ def test_load_service_config_from_file_no_dependencies(tmp_path: Path) -> None: "version": 0.1, "service_name": "example-service", "modes": {"default": []}, - } + }, + "services": {}, } create_config_file(tmp_path, config) @@ -126,7 +133,12 @@ def test_load_service_config_from_file_invalid_version(tmp_path: Path) -> None: "example-dependency": {"description": "Example dependency"} }, "modes": {"default": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -142,7 +154,12 @@ def test_load_service_config_from_file_missing_version(tmp_path: Path) -> None: "example-dependency": {"description": "Example dependency"} }, "modes": {"default": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -159,7 +176,12 @@ def test_load_service_config_from_file_missing_service_name(tmp_path: Path) -> N "example-dependency": {"description": "Example dependency"} }, "modes": {"default": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -177,7 +199,12 @@ def test_load_service_config_from_file_invalid_dependency(tmp_path: Path) -> Non "example-dependency": {"description": "Example dependency"} }, "modes": {"default": ["example-dependency", "unknown-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -198,7 +225,12 @@ def test_load_service_config_from_file_missing_default_mode(tmp_path: Path) -> N "example-dependency": {"description": "Example dependency"} }, "modes": {"custom": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -215,7 +247,12 @@ def test_load_service_config_from_file_no_modes(tmp_path: Path) -> None: "dependencies": { "example-dependency": {"description": "Example dependency"} }, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -224,6 +261,32 @@ def test_load_service_config_from_file_no_modes(tmp_path: Path) -> None: assert str(e.value) == "Default mode is required in service config" +def test_load_service_config_from_file_remote_dependency_not_in_services( + tmp_path: Path, +) -> None: + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "example-dependency": { + "description": "Example dependency", + "remote": { + "repo_name": "example-dependency", + "repo_link": "https://github.com/example/example-dependency", + "branch": "main", + }, + }, + }, + "modes": {"default": ["example-dependency"]}, + }, + "services": {}, + } + create_config_file(tmp_path, config) + + load_service_config_from_file(str(tmp_path)) + + def test_load_service_config_from_file_invalid_dependencies(tmp_path: Path) -> None: config = { "x-sentry-service-config": { @@ -236,7 +299,12 @@ def test_load_service_config_from_file_invalid_dependencies(tmp_path: Path) -> N } }, "modes": {"default": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -260,7 +328,12 @@ def test_load_service_config_from_file_invalid_modes(tmp_path: Path) -> None: "default": ["example-dependency"], "custom": "example-dependency", }, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) @@ -280,7 +353,12 @@ def test_load_service_config_from_file_no_x_sentry_service_config( "example-dependency": {"description": "Example dependency"} }, "modes": {"default": ["example-dependency"]}, - } + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, } create_config_file(tmp_path, config) From 6ecedd55a3b9c410a5ca527ac84b9fdfe3bcf7a9 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Mon, 12 May 2025 17:09:26 -0700 Subject: [PATCH 02/14] fix some tests --- devservices/utils/services.py | 4 ++++ tests/utils/test_services.py | 11 +++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/devservices/utils/services.py b/devservices/utils/services.py index 005bbd2b..6c96d9a7 100644 --- a/devservices/utils/services.py +++ b/devservices/utils/services.py @@ -8,6 +8,7 @@ from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError from devservices.exceptions import ServiceNotFoundError +from devservices.utils.console import Console from devservices.utils.devenv import get_coderoot @@ -22,12 +23,15 @@ def get_local_services(coderoot: str) -> list[Service]: """Get a list of services in the coderoot.""" from devservices.configs.service_config import load_service_config_from_file + console = Console() + services = [] for repo in os.listdir(coderoot): repo_path = os.path.join(coderoot, repo) try: service_config = load_service_config_from_file(repo_path) except (ConfigParseError, ConfigValidationError): + console.warning(f"{repo} was found with an invalid config") raise except ConfigNotFoundError: # Ignore repos that don't have devservices configs diff --git a/tests/utils/test_services.py b/tests/utils/test_services.py index e17f6a31..1f5a136f 100644 --- a/tests/utils/test_services.py +++ b/tests/utils/test_services.py @@ -7,6 +7,7 @@ import pytest from devservices.configs.service_config import ServiceConfig +from devservices.exceptions import ConfigParseError from devservices.exceptions import ServiceNotFoundError from devservices.utils.services import find_matching_service from devservices.utils.services import get_local_services @@ -32,13 +33,11 @@ def test_get_local_services_with_invalid_config( mock_repo_path = mock_code_root / "example" create_mock_git_repo("invalid_repo", mock_repo_path) - local_services = get_local_services(str(mock_code_root)) + with pytest.raises(ConfigParseError): + local_services = get_local_services(str(mock_code_root)) + assert not local_services captured = capsys.readouterr() - assert not local_services - assert ( - "example was found with an invalid config: Error parsing config file:" - in captured.out - ) + assert "example was found with an invalid config" in captured.out def test_get_local_services_with_valid_config(tmp_path: Path) -> None: From eccadb2505e3cea9351a115723fd773815802cd3 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 12:09:20 -0700 Subject: [PATCH 03/14] add supervisor config validation --- devservices/configs/service_config.py | 23 +++++++++- tests/configs/test_service_config.py | 64 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index 62e459b6..1cdfde54 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -5,12 +5,15 @@ from dataclasses import fields import yaml +from supervisor.options import ServerOptions from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.constants import PROGRAMS_CONF_FILE_NAME from devservices.exceptions import ConfigNotFoundError from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError +from devservices.utils.supervisor import SupervisorManager VALID_VERSIONS = [0.1] @@ -86,6 +89,10 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: docker_compose_services = config.get("services", {}).keys() + supervisor_programs = load_supervisor_programs_from_file( + repo_path, service_config_data.get("service_name") + ) + valid_dependency_keys = {field.name for field in fields(Dependency)} dependencies = {} @@ -113,9 +120,10 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: if ( dependency.remote is None and dependency_name not in docker_compose_services + and dependency_name not in supervisor_programs ): raise ConfigValidationError( - f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services" + f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" ) service_config = ServiceConfig( @@ -126,3 +134,16 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: ) return service_config + + +def load_supervisor_programs_from_file(repo_path: str, service_name: str) -> set[str]: + programs_config_path = os.path.join( + repo_path, DEVSERVICES_DIR_NAME, PROGRAMS_CONF_FILE_NAME + ) + if not os.path.exists(programs_config_path): + return set() + manager = SupervisorManager(programs_config_path, service_name=service_name) + opts = ServerOptions() + opts.configfile = manager.config_file_path + opts.process_config() + return set([program.name for program in opts.process_group_configs]) diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index e6797630..e63ff9f6 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -10,6 +10,7 @@ from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError from testing.utils import create_config_file +from testing.utils import create_programs_conf_file @pytest.mark.parametrize( @@ -412,3 +413,66 @@ def test_load_service_config_from_file_invalid_yaml_tag(tmp_path: Path) -> None: str(e.value) == f"Error parsing config file: could not determine a constructor for the tag 'tag:yaml.org,2002:invalid_tag'\n in \"{tmp_path / 'devservices' / 'config.yml'}\", line 7, column 19" ) + + +def test_load_service_config_from_file_no_programs_file(tmp_path: Path) -> None: + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "example-dependency": { + "description": "Example dependency", + }, + "example-program": { + "description": "Example program", + }, + }, + "modes": {"default": ["example-dependency", "example-program"]}, + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, + } + create_config_file(tmp_path, config) + + with pytest.raises(ConfigValidationError) as e: + load_service_config_from_file(str(tmp_path)) + assert ( + str(e.value) + == "Dependency 'example-program' is not remote but is not defined in docker-compose services or programs file" + ) + + +def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> None: + service_config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "example-dependency": { + "description": "Example dependency", + }, + "example-program": { + "description": "Example program", + }, + }, + "modes": {"default": ["example-dependency", "example-program"]}, + }, + "services": { + "example-dependency": { + "image": "example-dependency", + } + }, + } + create_config_file(tmp_path, service_config) + + programs_config = """[program:example-program] +command=echo "Hello, World!" +autostart=true +""" + create_programs_conf_file(tmp_path, programs_config) + + load_service_config_from_file(str(tmp_path)) From b1e2dc70e7f8527622be8a8b4ec4406029e98f07 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 12:13:06 -0700 Subject: [PATCH 04/14] add two more tests --- tests/configs/test_service_config.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index e63ff9f6..8d473de7 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -6,6 +6,7 @@ import pytest from devservices.configs.service_config import load_service_config_from_file +from devservices.configs.service_config import load_supervisor_programs_from_file from devservices.exceptions import ConfigNotFoundError from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError @@ -476,3 +477,19 @@ def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> No create_programs_conf_file(tmp_path, programs_config) load_service_config_from_file(str(tmp_path)) + + +def test_load_supervisor_programs_from_file_no_programs_file(tmp_path: Path) -> None: + programs = load_supervisor_programs_from_file(str(tmp_path), "example-service") + assert programs == set() + + +def test_load_supervisor_programs_from_file_valid_programs_file(tmp_path: Path) -> None: + programs_config = """[program:example-program] +command=echo "Hello, World!" +autostart=true +""" + create_programs_conf_file(tmp_path, programs_config) + + programs = load_supervisor_programs_from_file(str(tmp_path), "example-service") + assert programs == {"example-program"} From d9c5fc824ab64fb050c3ae3208ec06db6ff8f771 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 14:06:01 -0700 Subject: [PATCH 05/14] change structure of dependeny to also pass in type --- devservices/configs/service_config.py | 28 ++++++++++++++++++------- tests/configs/test_service_config.py | 30 +++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index 1cdfde54..b45b73a6 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -3,6 +3,7 @@ import os from dataclasses import dataclass from dataclasses import fields +from enum import Enum import yaml from supervisor.options import ServerOptions @@ -18,6 +19,11 @@ VALID_VERSIONS = [0.1] +class DependencyType(Enum): + DOCKER_COMPOSE = "docker-compose" + SUPERVISOR = "supervisor" + + @dataclass class RemoteConfig: repo_name: str @@ -30,6 +36,7 @@ class RemoteConfig: class Dependency: description: str remote: RemoteConfig | None = None + dependency_type: str | None = None @dataclass @@ -117,14 +124,19 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: # Validate that all non-remote dependencies are defined in docker-compose services for dependency_name, dependency in dependencies.items(): - if ( - dependency.remote is None - and dependency_name not in docker_compose_services - and dependency_name not in supervisor_programs - ): - raise ConfigValidationError( - f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" - ) + if dependency.remote is None: + if dependency_name in supervisor_programs: + dependencies[ + dependency_name + ].dependency_type = DependencyType.SUPERVISOR.value + elif dependency_name in docker_compose_services: + dependencies[ + dependency_name + ].dependency_type = DependencyType.DOCKER_COMPOSE.value + else: + raise ConfigValidationError( + f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" + ) service_config = ServiceConfig( version=service_config_data.get("version"), diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index 8d473de7..1b3f326e 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -5,6 +5,7 @@ import pytest +from devservices.configs.service_config import DependencyType from devservices.configs.service_config import load_service_config_from_file from devservices.configs.service_config import load_supervisor_programs_from_file from devservices.exceptions import ConfigNotFoundError @@ -15,12 +16,15 @@ @pytest.mark.parametrize( - "service_name, dependencies, modes", + "service_name, dependencies, modes, dependency_types", [ ( "example-service", {"example-dependency": {"description": "Example dependency"}}, {"default": ["example-dependency"]}, + { + "example-dependency": DependencyType.DOCKER_COMPOSE.value, + }, ), ( "example-service", @@ -39,6 +43,10 @@ }, }, {"default": ["example-dependency-1", "example-dependency-2"]}, + { + "example-dependency-1": None, + "example-dependency-2": DependencyType.DOCKER_COMPOSE.value, + }, ), ( "example-service", @@ -57,6 +65,10 @@ }, }, {"default": ["example-dependency-1"], "custom": ["example-dependency-2"]}, + { + "example-dependency-1": None, + "example-dependency-2": DependencyType.DOCKER_COMPOSE.value, + }, ), ], ) @@ -65,6 +77,7 @@ def test_load_service_config_from_file( service_name: str, dependencies: dict[str, dict[str, object]], modes: dict[str, list[str]], + dependency_types: dict[str, DependencyType], ) -> None: config = { "x-sentry-service-config": { @@ -90,6 +103,7 @@ def test_load_service_config_from_file( key: { "description": value["description"], "remote": value.get("remote"), + "dependency_type": dependency_types[key], } for key, value in dependencies.items() }, @@ -448,7 +462,7 @@ def test_load_service_config_from_file_no_programs_file(tmp_path: Path) -> None: def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> None: - service_config = { + devservices_config = { "x-sentry-service-config": { "version": 0.1, "service_name": "example-service", @@ -468,7 +482,7 @@ def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> No } }, } - create_config_file(tmp_path, service_config) + create_config_file(tmp_path, devservices_config) programs_config = """[program:example-program] command=echo "Hello, World!" @@ -476,7 +490,15 @@ def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> No """ create_programs_conf_file(tmp_path, programs_config) - load_service_config_from_file(str(tmp_path)) + service_config = load_service_config_from_file(str(tmp_path)) + assert ( + service_config.dependencies["example-program"].dependency_type + == DependencyType.SUPERVISOR.value + ) + assert ( + service_config.dependencies["example-dependency"].dependency_type + == DependencyType.DOCKER_COMPOSE.value + ) def test_load_supervisor_programs_from_file_no_programs_file(tmp_path: Path) -> None: From 7e4beff67936ee582a2afc569115fa49d9b27644 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 14:30:10 -0700 Subject: [PATCH 06/14] fix more stuff --- devservices/configs/service_config.py | 46 ++++++++++++--------------- devservices/constants.py | 7 ++++ devservices/utils/dependencies.py | 16 ++++------ tests/configs/test_service_config.py | 16 +++++----- 4 files changed, 42 insertions(+), 43 deletions(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index b45b73a6..bb91a7bc 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -3,12 +3,12 @@ import os from dataclasses import dataclass from dataclasses import fields -from enum import Enum import yaml from supervisor.options import ServerOptions from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.constants import PROGRAMS_CONF_FILE_NAME from devservices.exceptions import ConfigNotFoundError @@ -19,11 +19,6 @@ VALID_VERSIONS = [0.1] -class DependencyType(Enum): - DOCKER_COMPOSE = "docker-compose" - SUPERVISOR = "supervisor" - - @dataclass class RemoteConfig: repo_name: str @@ -35,8 +30,8 @@ class RemoteConfig: @dataclass class Dependency: description: str + dependency_type: DependencyType remote: RemoteConfig | None = None - dependency_type: str | None = None @dataclass @@ -111,33 +106,32 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: raise ConfigParseError( f"Unexpected key(s) in dependency '{key}': {unexpected_keys}" ) + if value.get("remote") is None: + if key in supervisor_programs: + dependency_type = DependencyType.SUPERVISOR + elif key in docker_compose_services: + dependency_type = DependencyType.COMPOSE + else: + raise ConfigValidationError( + f"Dependency '{key}' is not remote but is not defined in docker-compose services or programs file" + ) + else: + dependency_type = DependencyType.SERVICE + dependencies[key] = Dependency( description=value.get("description"), - remote=RemoteConfig(**value.get("remote")) - if "remote" in value - else None, + remote=( + RemoteConfig(**value.get("remote")) + if "remote" in value + else None + ), + dependency_type=dependency_type, ) except TypeError as type_error: raise ConfigParseError( f"Error parsing service dependencies: {type_error}" ) from type_error - # Validate that all non-remote dependencies are defined in docker-compose services - for dependency_name, dependency in dependencies.items(): - if dependency.remote is None: - if dependency_name in supervisor_programs: - dependencies[ - dependency_name - ].dependency_type = DependencyType.SUPERVISOR.value - elif dependency_name in docker_compose_services: - dependencies[ - dependency_name - ].dependency_type = DependencyType.DOCKER_COMPOSE.value - else: - raise ConfigValidationError( - f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" - ) - service_config = ServiceConfig( version=service_config_data.get("version"), service_name=service_config_data.get("service_name"), diff --git a/devservices/constants.py b/devservices/constants.py index 80e8433b..ce6b96ed 100644 --- a/devservices/constants.py +++ b/devservices/constants.py @@ -2,6 +2,7 @@ import os from datetime import timedelta +from enum import Enum class Color: @@ -15,6 +16,12 @@ class Color: RESET = "\033[0m" +class DependencyType(str, Enum): + SERVICE = "service" + COMPOSE = "compose" + SUPERVISOR = "supervisor" + + MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7" DEVSERVICES_DIR_NAME = "devservices" CONFIG_FILE_NAME = "config.yml" diff --git a/devservices/utils/dependencies.py b/devservices/utils/dependencies.py index 018fe714..dcae655f 100644 --- a/devservices/utils/dependencies.py +++ b/devservices/utils/dependencies.py @@ -10,7 +10,6 @@ from concurrent.futures import as_completed from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass -from enum import Enum from typing import TextIO from typing import TypeGuard @@ -24,6 +23,7 @@ from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEPENDENCY_CONFIG_VERSION from devservices.constants import DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR from devservices.constants import DEVSERVICES_DIR_NAME from devservices.constants import LOGGER_NAME @@ -55,11 +55,6 @@ ] -class DependencyType(str, Enum): - SERVICE = "service" - COMPOSE = "compose" - - @dataclass(frozen=True, eq=True) class DependencyNode: name: str @@ -751,6 +746,7 @@ def _construct_dependency_graph( for mode in modes: service_mode_dependencies.update(service_config.modes.get(mode, [])) for dependency_name, dependency in service_config.dependencies.items(): + print(dependency_name, dependency) # Skip the dependency if it's not in the modes (since it may not be installed and we don't care about it) if dependency_name not in service_mode_dependencies: continue @@ -761,9 +757,11 @@ def _construct_dependency_graph( ), DependencyNode( name=dependency_name, - dependency_type=DependencyType.SERVICE - if _has_remote_config(dependency.remote) - else DependencyType.COMPOSE, + dependency_type=( + DependencyType.SERVICE + if _has_remote_config(dependency.remote) + else dependency.dependency_type + ), ), ) if _has_remote_config(dependency.remote): diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index 1b3f326e..7809e0b3 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -5,9 +5,9 @@ import pytest -from devservices.configs.service_config import DependencyType from devservices.configs.service_config import load_service_config_from_file from devservices.configs.service_config import load_supervisor_programs_from_file +from devservices.constants import DependencyType from devservices.exceptions import ConfigNotFoundError from devservices.exceptions import ConfigParseError from devservices.exceptions import ConfigValidationError @@ -23,7 +23,7 @@ {"example-dependency": {"description": "Example dependency"}}, {"default": ["example-dependency"]}, { - "example-dependency": DependencyType.DOCKER_COMPOSE.value, + "example-dependency": DependencyType.COMPOSE, }, ), ( @@ -44,8 +44,8 @@ }, {"default": ["example-dependency-1", "example-dependency-2"]}, { - "example-dependency-1": None, - "example-dependency-2": DependencyType.DOCKER_COMPOSE.value, + "example-dependency-1": DependencyType.SERVICE, + "example-dependency-2": DependencyType.COMPOSE, }, ), ( @@ -66,8 +66,8 @@ }, {"default": ["example-dependency-1"], "custom": ["example-dependency-2"]}, { - "example-dependency-1": None, - "example-dependency-2": DependencyType.DOCKER_COMPOSE.value, + "example-dependency-1": DependencyType.SERVICE, + "example-dependency-2": DependencyType.COMPOSE, }, ), ], @@ -493,11 +493,11 @@ def test_load_service_config_from_file_valid_programs_file(tmp_path: Path) -> No service_config = load_service_config_from_file(str(tmp_path)) assert ( service_config.dependencies["example-program"].dependency_type - == DependencyType.SUPERVISOR.value + == DependencyType.SUPERVISOR ) assert ( service_config.dependencies["example-dependency"].dependency_type - == DependencyType.DOCKER_COMPOSE.value + == DependencyType.COMPOSE ) From eb39a558eef1403f8c7bb617b66f0d38c402efcd Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 14:30:52 -0700 Subject: [PATCH 07/14] remove print statement --- devservices/utils/dependencies.py | 1 - 1 file changed, 1 deletion(-) diff --git a/devservices/utils/dependencies.py b/devservices/utils/dependencies.py index dcae655f..bc1f01d2 100644 --- a/devservices/utils/dependencies.py +++ b/devservices/utils/dependencies.py @@ -746,7 +746,6 @@ def _construct_dependency_graph( for mode in modes: service_mode_dependencies.update(service_config.modes.get(mode, [])) for dependency_name, dependency in service_config.dependencies.items(): - print(dependency_name, dependency) # Skip the dependency if it's not in the modes (since it may not be installed and we don't care about it) if dependency_name not in service_mode_dependencies: continue From 6d226757dcb73300cac4b9c01c0c631b29ffffb0 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 14:42:34 -0700 Subject: [PATCH 08/14] fix tests --- tests/utils/test_dependencies.py | 38 +++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_dependencies.py b/tests/utils/test_dependencies.py index f5ae14ae..af229246 100644 --- a/tests/utils/test_dependencies.py +++ b/tests/utils/test_dependencies.py @@ -15,6 +15,7 @@ from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEPENDENCY_CONFIG_VERSION from devservices.constants import DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import DependencyError from devservices.exceptions import DependencyNotInstalledError @@ -23,7 +24,6 @@ from devservices.exceptions import ModeDoesNotExistError from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.dependencies import get_installed_remote_dependencies from devservices.utils.dependencies import get_non_shared_remote_dependencies from devservices.utils.dependencies import GitConfigManager @@ -203,6 +203,7 @@ def test_verify_local_dependencies_no_remote_dependencies(tmp_path: Path) -> Non ): dependency = Dependency( description="Test dependency", + dependency_type=DependencyType.COMPOSE, ) assert verify_local_dependencies([dependency]) @@ -221,6 +222,7 @@ def test_verify_local_dependencies_with_remote_dependencies(tmp_path: Path) -> N dependency = Dependency( description="Test dependency", remote=remote_config, + dependency_type=DependencyType.COMPOSE, ) assert not verify_local_dependencies([dependency]) @@ -255,6 +257,7 @@ def test_get_installed_remote_dependencies_single_dep_not_installed( branch="main", repo_link=f"file://{tmp_path / 'test-repo'}", ), + dependency_type=DependencyType.SERVICE, ) with pytest.raises(DependencyNotInstalledError): get_installed_remote_dependencies(dependencies=[mock_dependency]) @@ -273,6 +276,7 @@ def test_get_installed_remote_dependencies_single_dep_installed(tmp_path: Path) branch="main", repo_link=f"file://{tmp_path / 'test-repo'}", ), + dependency_type=DependencyType.SERVICE, ) installed_remote_dependencies_initial = install_dependencies([mock_dependency]) installed_remote_dependencies = get_installed_remote_dependencies( @@ -1963,6 +1967,7 @@ def test_install_dependencies_nested_dependency_file_contention(tmp_path: Path) branch="main", repo_link=f"file://{repo_a_path}", ), + dependency_type=DependencyType.SERVICE, ) repo_b_dependency = Dependency( description="repo b", @@ -1971,6 +1976,7 @@ def test_install_dependencies_nested_dependency_file_contention(tmp_path: Path) branch="main", repo_link=f"file://{repo_b_path}", ), + dependency_type=DependencyType.SERVICE, ) dependencies = [repo_a_dependency, repo_b_dependency] @@ -2080,6 +2086,7 @@ def test_get_non_shared_remote_dependencies_no_shared_dependencies( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ) }, modes={"default": ["dependency-1"]}, @@ -2136,6 +2143,7 @@ def test_get_non_shared_remote_dependencies_no_shared_dependencies( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ) }, modes={"default": ["dependency-1"]}, @@ -2171,6 +2179,7 @@ def test_get_non_shared_remote_dependencies_shared_dependencies( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ) }, modes={"default": ["dependency-1"]}, @@ -2200,6 +2209,7 @@ def test_get_non_shared_remote_dependencies_shared_dependencies( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ) ] ) @@ -2234,6 +2244,7 @@ def test_get_non_shared_remote_dependencies_shared_dependencies( repo_link="file://path/to/dependency-3", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "dependency-4": Dependency( description="dependency-4", @@ -2242,6 +2253,7 @@ def test_get_non_shared_remote_dependencies_shared_dependencies( repo_link="file://path/to/dependency-4", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-3"], "other": ["dependency-4"]}, @@ -2278,6 +2290,7 @@ def test_get_non_shared_remote_dependencies_nested_shared_dependencies( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "dependency-2": Dependency( description="dependency-2", @@ -2286,6 +2299,7 @@ def test_get_non_shared_remote_dependencies_nested_shared_dependencies( repo_link="file://path/to/dependency-2", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-1", "dependency-2"]}, @@ -2327,6 +2341,7 @@ def test_get_non_shared_remote_dependencies_nested_shared_dependencies( repo_link="file://path/to/dependency-3", branch="main", ), + dependency_type=DependencyType.SERVICE, ) ] ) @@ -2352,6 +2367,7 @@ def test_get_non_shared_remote_dependencies_nested_shared_dependencies( repo_link="file://path/to/dependency-3", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-3"]}, @@ -2388,6 +2404,7 @@ def test_get_non_shared_remote_dependencies_with_local_runtime_dependency( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "service-2": Dependency( description="service-2", @@ -2396,6 +2413,7 @@ def test_get_non_shared_remote_dependencies_with_local_runtime_dependency( repo_link="file://path/to/service-2", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-1", "service-2"]}, @@ -2455,6 +2473,7 @@ def test_get_non_shared_remote_dependencies_with_local_runtime_dependency( repo_link="file://path/to/dependency-3", branch="main", ), + dependency_type=DependencyType.SERVICE, ) ] ) @@ -2478,6 +2497,7 @@ def test_install_and_verify_dependencies_simple( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "dependency-2": Dependency( description="dependency-2", @@ -2486,6 +2506,7 @@ def test_install_and_verify_dependencies_simple( repo_link="file://path/to/dependency-2", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-1", "dependency-2"]}, @@ -2519,6 +2540,7 @@ def test_install_and_verify_dependencies_mode_simple( repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "dependency-2": Dependency( description="dependency-2", @@ -2527,6 +2549,7 @@ def test_install_and_verify_dependencies_mode_simple( repo_link="file://path/to/dependency-2", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={ @@ -2557,6 +2580,7 @@ def test_install_and_verify_dependencies_mode_does_not_exist(tmp_path: Path) -> repo_link="file://path/to/dependency-1", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "dependency-2": Dependency( description="dependency-2", @@ -2565,6 +2589,7 @@ def test_install_and_verify_dependencies_mode_does_not_exist(tmp_path: Path) -> repo_link="file://path/to/dependency-2", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["dependency-1", "dependency-2"]}, @@ -2615,6 +2640,7 @@ def test_construct_dependency_graph_simple( repo_link=f"file://{dependency_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={ @@ -2729,9 +2755,11 @@ def test_construct_dependency_graph_one_nested_dependency( repo_link=f"file://{parent_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "grandparent-service": Dependency( description="grandparent-service", + dependency_type=DependencyType.COMPOSE, ), }, modes={ @@ -2877,9 +2905,11 @@ def test_construct_dependency_graph_shared_dependency( repo_link=f"file://{parent_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "grandparent-service": Dependency( description="grandparent-service", + dependency_type=DependencyType.COMPOSE, ), "child-service": Dependency( description="child-service", @@ -2888,6 +2918,7 @@ def test_construct_dependency_graph_shared_dependency( repo_link=f"file://{child_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), }, modes={ @@ -3037,9 +3068,11 @@ def test_construct_dependency_graph_non_self_reference( repo_link=f"file://{parent_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "grandparent-service-container": Dependency( description="grandparent-service-container", + dependency_type=DependencyType.COMPOSE, ), }, modes={ @@ -3252,6 +3285,7 @@ def test_construct_dependency_graph_complex( repo_link=f"file://{child_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "grandparent-service": Dependency( description="grandparent-service", @@ -3260,9 +3294,11 @@ def test_construct_dependency_graph_complex( repo_link=f"file://{grandparent_service_repo_path}", branch="main", ), + dependency_type=DependencyType.SERVICE, ), "complex-service": Dependency( description="complex-service", + dependency_type=DependencyType.COMPOSE, ), }, modes={ From 03b77d411cf67368dde96445ffaeaf56ce1e06e8 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 13 May 2025 15:32:06 -0700 Subject: [PATCH 09/14] actually fix tests --- devservices/commands/down.py | 2 +- devservices/commands/reset.py | 2 +- devservices/commands/status.py | 2 +- devservices/commands/up.py | 2 +- tests/commands/test_down.py | 6 +++ tests/commands/test_list_dependencies.py | 9 ++++- tests/commands/test_logs.py | 17 +++++++-- tests/commands/test_status.py | 22 ++++++++--- tests/commands/test_toggle.py | 48 +++++++++++++++++++++--- 9 files changed, 90 insertions(+), 20 deletions(-) diff --git a/devservices/commands/down.py b/devservices/commands/down.py index 8d34134c..0836ecb6 100644 --- a/devservices/commands/down.py +++ b/devservices/commands/down.py @@ -11,6 +11,7 @@ from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEPENDENCY_CONFIG_VERSION +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY from devservices.constants import DEVSERVICES_DIR_NAME @@ -23,7 +24,6 @@ from devservices.utils.console import Status from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.dependencies import get_non_shared_remote_dependencies from devservices.utils.dependencies import install_and_verify_dependencies from devservices.utils.dependencies import InstalledRemoteDependency diff --git a/devservices/commands/reset.py b/devservices/commands/reset.py index 1e936c5f..f0c909a0 100644 --- a/devservices/commands/reset.py +++ b/devservices/commands/reset.py @@ -7,6 +7,7 @@ from sentry_sdk import capture_exception from devservices.commands.down import down +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL from devservices.exceptions import DockerDaemonNotRunningError from devservices.exceptions import DockerError @@ -14,7 +15,6 @@ from devservices.utils.console import Status from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.docker import get_matching_containers from devservices.utils.docker import get_volumes_for_containers from devservices.utils.docker import remove_docker_resources diff --git a/devservices/commands/status.py b/devservices/commands/status.py index f04f4f58..59db155b 100644 --- a/devservices/commands/status.py +++ b/devservices/commands/status.py @@ -15,6 +15,7 @@ from devservices.constants import Color from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEPENDENCY_CONFIG_VERSION +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY from devservices.constants import DEVSERVICES_DIR_NAME @@ -27,7 +28,6 @@ from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import DependencyGraph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.dependencies import install_and_verify_dependencies from devservices.utils.dependencies import InstalledRemoteDependency from devservices.utils.docker_compose import get_docker_compose_commands_to_run diff --git a/devservices/commands/up.py b/devservices/commands/up.py index 0d3cb765..23a0014a 100644 --- a/devservices/commands/up.py +++ b/devservices/commands/up.py @@ -11,6 +11,7 @@ from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEPENDENCY_CONFIG_VERSION +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY from devservices.constants import DEVSERVICES_DIR_NAME @@ -25,7 +26,6 @@ from devservices.utils.console import Status from devservices.utils.dependencies import construct_dependency_graph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.dependencies import install_and_verify_dependencies from devservices.utils.dependencies import InstalledRemoteDependency from devservices.utils.docker import check_all_containers_healthy diff --git a/tests/commands/test_down.py b/tests/commands/test_down.py index 36b7ee67..125c205a 100644 --- a/tests/commands/test_down.py +++ b/tests/commands/test_down.py @@ -13,6 +13,7 @@ from devservices.configs.service_config import RemoteConfig from devservices.configs.service_config import ServiceConfig from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import ConfigError from devservices.exceptions import ServiceNotFoundError @@ -632,6 +633,7 @@ def test_down_does_not_stop_service_being_used_by_another_service( dependencies={ "redis": Dependency( description="Redis", + dependency_type=DependencyType.SERVICE, remote=RemoteConfig( repo_name="redis", repo_link=f"file://{redis_repo_path}", @@ -641,6 +643,7 @@ def test_down_does_not_stop_service_being_used_by_another_service( ), "example-service": Dependency( description="Example service", + dependency_type=DependencyType.SERVICE, remote=RemoteConfig( repo_name="example-service", repo_link=f"file://{example_repo_path}", @@ -814,6 +817,7 @@ def test_down_does_not_stop_nested_service_being_used_by_another_service( dependencies={ "parent-service": Dependency( description="Parent service", + dependency_type=DependencyType.SERVICE, remote=RemoteConfig( repo_name="parent-service", repo_link=f"file://{parent_repo_path}", @@ -1094,6 +1098,7 @@ def test_down_local_service_with_dependent_service_running( dependencies={ "redis": Dependency( description="Redis", + dependency_type=DependencyType.SERVICE, remote=RemoteConfig( repo_name="redis", repo_link=f"file://{redis_repo_path}", @@ -1103,6 +1108,7 @@ def test_down_local_service_with_dependent_service_running( ), "local-runtime-service": Dependency( description="Local runtime service", + dependency_type=DependencyType.SERVICE, remote=RemoteConfig( repo_name="local-runtime-service", repo_link=f"file://{local_runtime_repo_path}", diff --git a/tests/commands/test_list_dependencies.py b/tests/commands/test_list_dependencies.py index d866d6a1..84e247fd 100644 --- a/tests/commands/test_list_dependencies.py +++ b/tests/commands/test_list_dependencies.py @@ -10,6 +10,7 @@ from devservices.commands.list_dependencies import list_dependencies from devservices.configs.service_config import Dependency from devservices.configs.service_config import ServiceConfig +from devservices.constants import DependencyType from devservices.exceptions import ConfigValidationError from devservices.exceptions import ServiceNotFoundError from devservices.utils.services import Service @@ -115,8 +116,12 @@ def test_list_dependencies_with_dependencies( version=0.1, service_name="test-service", dependencies={ - "redis": Dependency(description="Redis"), - "postgres": Dependency(description="Postgres"), + "redis": Dependency( + description="Redis", dependency_type=DependencyType.COMPOSE + ), + "postgres": Dependency( + description="Postgres", dependency_type=DependencyType.COMPOSE + ), }, modes={"default": ["redis", "postgres"]}, ), diff --git a/tests/commands/test_logs.py b/tests/commands/test_logs.py index b5418ef0..9eda8ad5 100644 --- a/tests/commands/test_logs.py +++ b/tests/commands/test_logs.py @@ -12,6 +12,7 @@ from devservices.configs.service_config import Dependency from devservices.configs.service_config import ServiceConfig from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import ConfigError from devservices.exceptions import ServiceNotFoundError @@ -42,8 +43,12 @@ def test_logs_no_specified_service_not_running( version=0.1, service_name="example-service", dependencies={ - "redis": Dependency(description="Redis"), - "clickhouse": Dependency(description="Clickhouse"), + "redis": Dependency( + description="Redis", dependency_type=DependencyType.COMPOSE + ), + "clickhouse": Dependency( + description="Clickhouse", dependency_type=DependencyType.COMPOSE + ), }, modes={"default": ["redis", "clickhouse"]}, ), @@ -98,8 +103,12 @@ def test_logs_no_specified_service_success( version=0.1, service_name="example-service", dependencies={ - "redis": Dependency(description="Redis"), - "clickhouse": Dependency(description="Clickhouse"), + "redis": Dependency( + description="Redis", dependency_type=DependencyType.COMPOSE + ), + "clickhouse": Dependency( + description="Clickhouse", dependency_type=DependencyType.COMPOSE + ), }, modes={"default": ["redis", "clickhouse"]}, ), diff --git a/tests/commands/test_status.py b/tests/commands/test_status.py index cf21a3c5..52a577b3 100644 --- a/tests/commands/test_status.py +++ b/tests/commands/test_status.py @@ -20,13 +20,13 @@ from devservices.configs.service_config import ServiceConfig from devservices.constants import Color from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import DependencyError from devservices.exceptions import DockerComposeError from devservices.exceptions import ServiceNotFoundError from devservices.utils.dependencies import DependencyGraph from devservices.utils.dependencies import DependencyNode -from devservices.utils.dependencies import DependencyType from devservices.utils.services import Service from devservices.utils.state import State from devservices.utils.state import StateTables @@ -79,8 +79,14 @@ def test_get_status_json_results( version=0.1, service_name="test-service", dependencies={ - "redis": Dependency(description="Redis"), - "clickhouse": Dependency(description="Clickhouse"), + "redis": Dependency( + description="Redis", + dependency_type=DependencyType.COMPOSE, + ), + "clickhouse": Dependency( + description="Clickhouse", + dependency_type=DependencyType.COMPOSE, + ), }, modes={"default": ["redis", "clickhouse"], "test": ["redis"]}, ), @@ -541,8 +547,14 @@ def test_handle_started_service( version=0.1, service_name="test-service", dependencies={ - "redis": Dependency(description="Redis"), - "clickhouse": Dependency(description="Clickhouse"), + "redis": Dependency( + description="Redis", + dependency_type=DependencyType.COMPOSE, + ), + "clickhouse": Dependency( + description="Clickhouse", + dependency_type=DependencyType.COMPOSE, + ), }, modes={"default": ["redis", "clickhouse"], "test": ["redis"]}, ), diff --git a/tests/commands/test_toggle.py b/tests/commands/test_toggle.py index f397ee61..f15752f2 100644 --- a/tests/commands/test_toggle.py +++ b/tests/commands/test_toggle.py @@ -17,6 +17,7 @@ from devservices.configs.service_config import RemoteConfig from devservices.configs.service_config import ServiceConfig from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DependencyType from devservices.constants import DEVSERVICES_DIR_NAME from devservices.exceptions import CannotToggleNonRemoteServiceError from devservices.exceptions import ConfigNotFoundError @@ -333,6 +334,7 @@ def test_toggle_dependent_service_running( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), "example-service": Dependency( description="Example service", @@ -342,6 +344,7 @@ def test_toggle_dependent_service_running( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["redis", "example-service"]}, @@ -416,6 +419,7 @@ def test_toggle_to_local_runtime_no_runtime_specified( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -439,6 +443,7 @@ def test_toggle_to_local_runtime_no_runtime_specified( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -468,6 +473,7 @@ def test_toggle_cannot_toggle_non_remote_service( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -497,6 +503,7 @@ def test_toggle_cannot_toggle_non_remote_service( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -536,6 +543,7 @@ def test_handle_transition_to_local_runtime_currently_running_standalone( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -616,6 +624,7 @@ def test_handle_transition_to_local_runtime_naming_conflict( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -640,6 +649,7 @@ def test_handle_transition_to_local_runtime_naming_conflict( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -694,6 +704,7 @@ def test_handle_transition_to_containerized_runtime_no_dependent_services( "redis": Dependency( description="Redis", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["redis"]}, @@ -745,7 +756,11 @@ def test_handle_transition_to_containerized_runtime_with_service_running( version=0.1, service_name="redis", dependencies={ - "redis": Dependency(description="Redis", remote=None), + "redis": Dependency( + description="Redis", + remote=None, + dependency_type=DependencyType.COMPOSE, + ), }, modes={"default": ["redis"]}, ), @@ -872,6 +887,7 @@ def test_handle_transition_to_containerized_runtime_with_dependent_services( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), "example-service": Dependency( description="Example service", @@ -881,6 +897,7 @@ def test_handle_transition_to_containerized_runtime_with_dependent_services( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["redis", "example-service"]}, @@ -911,10 +928,12 @@ def test_handle_transition_to_containerized_runtime_with_dependent_services( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["redis", "clickhouse"]}, @@ -1162,6 +1181,7 @@ def test_bring_down_containerized_service_install_and_verify_dependencies_failur "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -1214,6 +1234,7 @@ def test_bring_down_containerized_service_no_remote_dependencies( "clickhouse": Dependency( description="Clickhouse", remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, @@ -1230,7 +1251,11 @@ def test_bring_down_containerized_service_no_remote_dependencies( version=0.1, service_name="example-service", dependencies={ - "clickhouse": Dependency(description="Clickhouse", remote=None), + "clickhouse": Dependency( + description="Clickhouse", + remote=None, + dependency_type=DependencyType.COMPOSE, + ), }, modes={"default": ["clickhouse"]}, ), @@ -1319,7 +1344,11 @@ def test_bring_down_containerized_service_with_remote_dependency( version=0.1, service_name="example-service", dependencies={ - "clickhouse": Dependency(description="Clickhouse", remote=None), + "clickhouse": Dependency( + description="Clickhouse", + remote=None, + dependency_type=DependencyType.COMPOSE, + ), "redis": Dependency( description="Redis", remote=RemoteConfig( @@ -1328,6 +1357,7 @@ def test_bring_down_containerized_service_with_remote_dependency( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["clickhouse", "redis"]}, @@ -1347,7 +1377,9 @@ def test_bring_down_containerized_service_with_remote_dependency( service_name="example-service", dependencies={ "clickhouse": Dependency( - description="Clickhouse", remote=None + description="Clickhouse", + remote=None, + dependency_type=DependencyType.COMPOSE, ), "redis": Dependency( description="Redis", @@ -1357,6 +1389,7 @@ def test_bring_down_containerized_service_with_remote_dependency( branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["clickhouse", "redis"]}, @@ -1410,6 +1443,7 @@ def test_bring_down_containerized_service_get_non_shared_remote_dependencies_err branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["redis"]}, @@ -1434,6 +1468,7 @@ def test_bring_down_containerized_service_get_non_shared_remote_dependencies_err branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["redis"]}, @@ -1458,6 +1493,7 @@ def test_bring_down_containerized_service_get_non_shared_remote_dependencies_err branch="main", mode="default", ), + dependency_type=DependencyType.SERVICE, ), }, modes={"default": ["redis"]}, @@ -1519,7 +1555,9 @@ def test_bring_down_containerized_service_docker_compose_error( service_name="example-service", dependencies={ "clickhouse": Dependency( - description="Clickhouse", remote=None + description="Clickhouse", + remote=None, + dependency_type=DependencyType.COMPOSE, ), }, modes={"default": ["clickhouse"]}, From c92b0c16639910ce9bf76b899a6435aa9e1e623d Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 14 May 2025 09:39:16 -0700 Subject: [PATCH 10/14] fix dependency_type --- devservices/utils/dependencies.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/devservices/utils/dependencies.py b/devservices/utils/dependencies.py index bc1f01d2..6fe0b8d8 100644 --- a/devservices/utils/dependencies.py +++ b/devservices/utils/dependencies.py @@ -755,12 +755,7 @@ def _construct_dependency_graph( dependency_type=DependencyType.SERVICE, ), DependencyNode( - name=dependency_name, - dependency_type=( - DependencyType.SERVICE - if _has_remote_config(dependency.remote) - else dependency.dependency_type - ), + name=dependency_name, dependency_type=dependency.dependency_type ), ) if _has_remote_config(dependency.remote): From d5055119ff1b190b977ef02c0b552133e0206a0c Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 21 May 2025 09:41:39 -0700 Subject: [PATCH 11/14] change to StrEnum --- devservices/constants.py | 4 ++-- devservices/utils/state.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/devservices/constants.py b/devservices/constants.py index ce6b96ed..bdbd7666 100644 --- a/devservices/constants.py +++ b/devservices/constants.py @@ -2,7 +2,7 @@ import os from datetime import timedelta -from enum import Enum +from enum import StrEnum class Color: @@ -16,7 +16,7 @@ class Color: RESET = "\033[0m" -class DependencyType(str, Enum): +class DependencyType(StrEnum): SERVICE = "service" COMPOSE = "compose" SUPERVISOR = "supervisor" diff --git a/devservices/utils/state.py b/devservices/utils/state.py index 8f11189f..3c3f1d67 100644 --- a/devservices/utils/state.py +++ b/devservices/utils/state.py @@ -3,13 +3,14 @@ import os import sqlite3 from enum import Enum +from enum import StrEnum from typing import Literal from devservices.constants import DEVSERVICES_LOCAL_DIR from devservices.constants import STATE_DB_FILE -class ServiceRuntime(str, Enum): +class ServiceRuntime(StrEnum): LOCAL = "local" CONTAINERIZED = "containerized" From f2c98e7b41496a85451c826a89f5304510f8550a Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 21 May 2025 09:58:00 -0700 Subject: [PATCH 12/14] fix tests --- tests/commands/test_list_services.py | 2 +- tests/configs/test_service_config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_list_services.py b/tests/commands/test_list_services.py index ed0b8360..8baa76c0 100644 --- a/tests/commands/test_list_services.py +++ b/tests/commands/test_list_services.py @@ -161,7 +161,7 @@ def test_list_running_services_config_error( assert ( captured.out - == "\x1b[0;33mexample-service was found with an invalid config\x1b[0m\n\x1b[0;31mDependency 'clickhouse' is not remote but is not defined in docker-compose services\x1b[0m\n" + == "\x1b[0;33mexample-service was found with an invalid config\x1b[0m\n\x1b[0;31mDependency 'clickhouse' is not remote but is not defined in docker-compose services or programs file\x1b[0m\n" ) diff --git a/tests/configs/test_service_config.py b/tests/configs/test_service_config.py index 2e505dfc..695a3ab9 100644 --- a/tests/configs/test_service_config.py +++ b/tests/configs/test_service_config.py @@ -325,7 +325,7 @@ def test_load_service_config_from_file_no_matching_docker_compose_service( load_service_config_from_file(str(tmp_path)) assert ( str(e.value) - == "Dependency 'example-dependency' is not remote but is not defined in docker-compose services" + == "Dependency 'example-dependency' is not remote but is not defined in docker-compose services or programs file" ) From 5626fecf2cd278f3265fedd99cd76c332a3ba732 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 21 May 2025 10:11:37 -0700 Subject: [PATCH 13/14] fix tests again --- devservices/configs/service_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index 6a430fdb..bc6d810b 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -139,7 +139,7 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: and dependency_name not in docker_compose_services ): raise ConfigValidationError( - f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services" + f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" ) service_config = ServiceConfig( From e768dc00b39363ebf56328b846c34b5a69119a71 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Wed, 21 May 2025 10:20:25 -0700 Subject: [PATCH 14/14] should not have been merged in main --- devservices/configs/service_config.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/devservices/configs/service_config.py b/devservices/configs/service_config.py index bc6d810b..3b6a32de 100644 --- a/devservices/configs/service_config.py +++ b/devservices/configs/service_config.py @@ -108,6 +108,7 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: ) if value.get("remote") is None: if key in supervisor_programs: + print("lol") dependency_type = DependencyType.SUPERVISOR elif key in docker_compose_services: dependency_type = DependencyType.COMPOSE @@ -132,16 +133,6 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig: f"Error parsing service dependencies: {type_error}" ) from type_error - # Validate that all non-remote dependencies are defined in docker-compose services - for dependency_name, dependency in dependencies.items(): - if ( - dependency.remote is None - and dependency_name not in docker_compose_services - ): - raise ConfigValidationError( - f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services or programs file" - ) - service_config = ServiceConfig( version=service_config_data.get("version"), service_name=service_config_data.get("service_name"),