feat(config): Add supervisor programs dependency validation

devservices/commands/down.py

@@ -11,6 +11,7 @@
 
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
@@ -23,7 +24,6 @@
 from devservices.utils.console import Status
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import DependencyNode
-from devservices.utils.dependencies import DependencyType
 from devservices.utils.dependencies import get_non_shared_remote_dependencies
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency

devservices/commands/reset.py

@@ -7,14 +7,14 @@
 from sentry_sdk import capture_exception
 
 from devservices.commands.down import down
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_ORCHESTRATOR_LABEL
 from devservices.exceptions import DockerDaemonNotRunningError
 from devservices.exceptions import DockerError
 from devservices.utils.console import Console
 from devservices.utils.console import Status
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import DependencyNode
-from devservices.utils.dependencies import DependencyType
 from devservices.utils.docker import get_matching_containers
 from devservices.utils.docker import get_volumes_for_containers
 from devservices.utils.docker import remove_docker_resources

devservices/commands/status.py

@@ -15,6 +15,7 @@
 from devservices.constants import Color
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
@@ -27,7 +28,6 @@
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import DependencyGraph
 from devservices.utils.dependencies import DependencyNode
-from devservices.utils.dependencies import DependencyType
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.docker_compose import get_docker_compose_commands_to_run

devservices/commands/up.py

@@ -11,6 +11,7 @@
 
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR_KEY
 from devservices.constants import DEVSERVICES_DIR_NAME
@@ -25,7 +26,6 @@
 from devservices.utils.console import Status
 from devservices.utils.dependencies import construct_dependency_graph
 from devservices.utils.dependencies import DependencyNode
-from devservices.utils.dependencies import DependencyType
 from devservices.utils.dependencies import install_and_verify_dependencies
 from devservices.utils.dependencies import InstalledRemoteDependency
 from devservices.utils.docker import check_all_containers_healthy

devservices/configs/service_config.py

@@ -5,12 +5,16 @@
 from dataclasses import fields
 
 import yaml
+from supervisor.options import ServerOptions
 
 from devservices.constants import CONFIG_FILE_NAME
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DIR_NAME
+from devservices.constants import PROGRAMS_CONF_FILE_NAME
 from devservices.exceptions import ConfigNotFoundError
 from devservices.exceptions import ConfigParseError
 from devservices.exceptions import ConfigValidationError
+from devservices.utils.supervisor import SupervisorManager
 
 VALID_VERSIONS = [0.1]
 
@@ -26,6 +30,7 @@ class RemoteConfig:
 @dataclass
 class Dependency:
     description: str
+    dependency_type: DependencyType
     remote: RemoteConfig | None = None
 
 
@@ -86,6 +91,10 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
 
         docker_compose_services = config.get("services", {}).keys()
 
+        supervisor_programs = load_supervisor_programs_from_file(
+            repo_path, service_config_data.get("service_name")
+        )
+
         valid_dependency_keys = {field.name for field in fields(Dependency)}
 
         dependencies = {}
@@ -97,27 +106,33 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
                     raise ConfigParseError(
                         f"Unexpected key(s) in dependency '{key}': {unexpected_keys}"
                     )
+                if value.get("remote") is None:
+                    if key in supervisor_programs:
+                        print("lol")
+                        dependency_type = DependencyType.SUPERVISOR
+                    elif key in docker_compose_services:
+                        dependency_type = DependencyType.COMPOSE
+                    else:
+                        raise ConfigValidationError(
+                            f"Dependency '{key}' is not remote but is not defined in docker-compose services or programs file"
+                        )
+                else:
+                    dependency_type = DependencyType.SERVICE
+
                 dependencies[key] = Dependency(
                     description=value.get("description"),
-                    remote=RemoteConfig(**value.get("remote"))
-                    if "remote" in value
-                    else None,
+                    remote=(
+                        RemoteConfig(**value.get("remote"))
+                        if "remote" in value
+                        else None
+                    ),
+                    dependency_type=dependency_type,
                 )
         except TypeError as type_error:
             raise ConfigParseError(
                 f"Error parsing service dependencies: {type_error}"
             ) from type_error
 
-        # Validate that all non-remote dependencies are defined in docker-compose services
-        for dependency_name, dependency in dependencies.items():
-            if (
-                dependency.remote is None
-                and dependency_name not in docker_compose_services
-            ):
-                raise ConfigValidationError(
-                    f"Dependency '{dependency_name}' is not remote but is not defined in docker-compose services"
-                )
-
         service_config = ServiceConfig(
             version=service_config_data.get("version"),
             service_name=service_config_data.get("service_name"),
@@ -126,3 +141,16 @@ def load_service_config_from_file(repo_path: str) -> ServiceConfig:
         )
 
         return service_config
+
+
+def load_supervisor_programs_from_file(repo_path: str, service_name: str) -> set[str]:
+    programs_config_path = os.path.join(
+        repo_path, DEVSERVICES_DIR_NAME, PROGRAMS_CONF_FILE_NAME
+    )
+    if not os.path.exists(programs_config_path):
+        return set()
+    manager = SupervisorManager(programs_config_path, service_name=service_name)
+    opts = ServerOptions()
+    opts.configfile = manager.config_file_path
+    opts.process_config()
+    return set([program.name for program in opts.process_group_configs])

devservices/constants.py

@@ -2,6 +2,7 @@
 
 import os
 from datetime import timedelta
+from enum import StrEnum
 
 
 class Color:
@@ -15,6 +16,12 @@ class Color:
     RESET = "\033[0m"
 
 
+class DependencyType(StrEnum):
+    SERVICE = "service"
+    COMPOSE = "compose"
+    SUPERVISOR = "supervisor"
+
+
 MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7"
 DEVSERVICES_DIR_NAME = "devservices"
 CONFIG_FILE_NAME = "config.yml"

devservices/utils/dependencies.py

@@ -10,7 +10,6 @@
 from concurrent.futures import as_completed
 from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
-from enum import Enum
 from typing import TextIO
 from typing import TypeGuard
 
@@ -24,6 +23,7 @@
 from devservices.constants import CONFIG_FILE_NAME
 from devservices.constants import DEPENDENCY_CONFIG_VERSION
 from devservices.constants import DEPENDENCY_GIT_PARTIAL_CLONE_CONFIG_OPTIONS
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DEPENDENCIES_CACHE_DIR
 from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.constants import LOGGER_NAME
@@ -55,11 +55,6 @@
 ]
 
 
-class DependencyType(str, Enum):
-    SERVICE = "service"
-    COMPOSE = "compose"
-
-
 @dataclass(frozen=True, eq=True)
 class DependencyNode:
     name: str
@@ -760,10 +755,7 @@ def _construct_dependency_graph(
                     dependency_type=DependencyType.SERVICE,
                 ),
                 DependencyNode(
-                    name=dependency_name,
-                    dependency_type=DependencyType.SERVICE
-                    if _has_remote_config(dependency.remote)
-                    else DependencyType.COMPOSE,
+                    name=dependency_name, dependency_type=dependency.dependency_type
                 ),
             )
             if _has_remote_config(dependency.remote):

devservices/utils/state.py

@@ -3,13 +3,14 @@
 import os
 import sqlite3
 from enum import Enum
+from enum import StrEnum
 from typing import Literal
 
 from devservices.constants import DEVSERVICES_LOCAL_DIR
 from devservices.constants import STATE_DB_FILE
 
 
-class ServiceRuntime(str, Enum):
+class ServiceRuntime(StrEnum):
     LOCAL = "local"
     CONTAINERIZED = "containerized"
 

tests/commands/test_down.py

@@ -13,6 +13,7 @@
 from devservices.configs.service_config import RemoteConfig
 from devservices.configs.service_config import ServiceConfig
 from devservices.constants import CONFIG_FILE_NAME
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.exceptions import ConfigError
 from devservices.exceptions import ServiceNotFoundError
@@ -632,6 +633,7 @@ def test_down_does_not_stop_service_being_used_by_another_service(
                     dependencies={
                         "redis": Dependency(
                             description="Redis",
+                            dependency_type=DependencyType.SERVICE,
                             remote=RemoteConfig(
                                 repo_name="redis",
                                 repo_link=f"file://{redis_repo_path}",
@@ -641,6 +643,7 @@ def test_down_does_not_stop_service_being_used_by_another_service(
                         ),
                         "example-service": Dependency(
                             description="Example service",
+                            dependency_type=DependencyType.SERVICE,
                             remote=RemoteConfig(
                                 repo_name="example-service",
                                 repo_link=f"file://{example_repo_path}",
@@ -814,6 +817,7 @@ def test_down_does_not_stop_nested_service_being_used_by_another_service(
                     dependencies={
                         "parent-service": Dependency(
                             description="Parent service",
+                            dependency_type=DependencyType.SERVICE,
                             remote=RemoteConfig(
                                 repo_name="parent-service",
                                 repo_link=f"file://{parent_repo_path}",
@@ -1094,6 +1098,7 @@ def test_down_local_service_with_dependent_service_running(
                     dependencies={
                         "redis": Dependency(
                             description="Redis",
+                            dependency_type=DependencyType.SERVICE,
                             remote=RemoteConfig(
                                 repo_name="redis",
                                 repo_link=f"file://{redis_repo_path}",
@@ -1103,6 +1108,7 @@ def test_down_local_service_with_dependent_service_running(
                         ),
                         "local-runtime-service": Dependency(
                             description="Local runtime service",
+                            dependency_type=DependencyType.SERVICE,
                             remote=RemoteConfig(
                                 repo_name="local-runtime-service",
                                 repo_link=f"file://{local_runtime_repo_path}",

tests/commands/test_list_dependencies.py

@@ -10,6 +10,7 @@
 from devservices.commands.list_dependencies import list_dependencies
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import ServiceConfig
+from devservices.constants import DependencyType
 from devservices.exceptions import ConfigValidationError
 from devservices.exceptions import ServiceNotFoundError
 from devservices.utils.services import Service
@@ -115,8 +116,12 @@ def test_list_dependencies_with_dependencies(
             version=0.1,
             service_name="test-service",
             dependencies={
-                "redis": Dependency(description="Redis"),
-                "postgres": Dependency(description="Postgres"),
+                "redis": Dependency(
+                    description="Redis", dependency_type=DependencyType.COMPOSE
+                ),
+                "postgres": Dependency(
+                    description="Postgres", dependency_type=DependencyType.COMPOSE
+                ),
             },
             modes={"default": ["redis", "postgres"]},
         ),

tests/commands/test_list_services.py

@@ -161,7 +161,7 @@ def test_list_running_services_config_error(
 
         assert (
             captured.out
-            == "\x1b[0;33mexample-service was found with an invalid config\x1b[0m\n\x1b[0;31mDependency 'clickhouse' is not remote but is not defined in docker-compose services\x1b[0m\n"
+            == "\x1b[0;33mexample-service was found with an invalid config\x1b[0m\n\x1b[0;31mDependency 'clickhouse' is not remote but is not defined in docker-compose services or programs file\x1b[0m\n"
         )
 
 

tests/commands/test_logs.py

@@ -12,6 +12,7 @@
 from devservices.configs.service_config import Dependency
 from devservices.configs.service_config import ServiceConfig
 from devservices.constants import CONFIG_FILE_NAME
+from devservices.constants import DependencyType
 from devservices.constants import DEVSERVICES_DIR_NAME
 from devservices.exceptions import ConfigError
 from devservices.exceptions import ServiceNotFoundError
@@ -42,8 +43,12 @@ def test_logs_no_specified_service_not_running(
                 version=0.1,
                 service_name="example-service",
                 dependencies={
-                    "redis": Dependency(description="Redis"),
-                    "clickhouse": Dependency(description="Clickhouse"),
+                    "redis": Dependency(
+                        description="Redis", dependency_type=DependencyType.COMPOSE
+                    ),
+                    "clickhouse": Dependency(
+                        description="Clickhouse", dependency_type=DependencyType.COMPOSE
+                    ),
                 },
                 modes={"default": ["redis", "clickhouse"]},
             ),
@@ -98,8 +103,12 @@ def test_logs_no_specified_service_success(
                 version=0.1,
                 service_name="example-service",
                 dependencies={
-                    "redis": Dependency(description="Redis"),
-                    "clickhouse": Dependency(description="Clickhouse"),
+                    "redis": Dependency(
+                        description="Redis", dependency_type=DependencyType.COMPOSE
+                    ),
+                    "clickhouse": Dependency(
+                        description="Clickhouse", dependency_type=DependencyType.COMPOSE
+                    ),
                 },
                 modes={"default": ["redis", "clickhouse"]},
             ),