From 8972001523e7db8b3ab1b98b6032066d00962cc8 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Fri, 18 Apr 2025 17:21:10 -0700 Subject: [PATCH 1/4] add serve command --- devservices/commands/serve.py | 49 +++++++++++++++++++++++++++++++++ devservices/main.py | 2 ++ devservices/utils/supervisor.py | 13 +++++++++ 3 files changed, 64 insertions(+) create mode 100644 devservices/commands/serve.py diff --git a/devservices/commands/serve.py b/devservices/commands/serve.py new file mode 100644 index 00000000..b90a8e48 --- /dev/null +++ b/devservices/commands/serve.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import os +import pty +import shlex +from argparse import _SubParsersAction +from argparse import ArgumentParser +from argparse import Namespace + +from sentry_sdk import capture_exception + +from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.exceptions import ConfigNotFoundError +from devservices.utils.console import Console +from devservices.utils.services import find_matching_service +from devservices.utils.supervisor import SupervisorManager + + +def add_parser(subparsers: _SubParsersAction[ArgumentParser]) -> None: + # prefix_chars is a hack to allow all options to be passed through to the devserver without argparse complaining + parser = subparsers.add_parser( + "serve", help="Serve the devserver", prefix_chars="+" + ) + parser.add_argument( + "extra", nargs="*", help="Flags to pass through to the devserver" + ) + parser.set_defaults(func=serve) + + +def serve(args: Namespace) -> None: + """Serve the devserver.""" + console = Console() + + try: + service = find_matching_service() + except ConfigNotFoundError as e: + capture_exception(e, level="info") + console.failure( + f"{str(e)}. Please specify a service (i.e. `devservices up sentry`) or run the command from a directory with a devservices configuration." + ) + exit(1) + + manager = SupervisorManager( + os.path.join(service.repo_path, f"{DEVSERVICES_DIR_NAME}/processes.conf"), + service_name=service.name, + ) + devserver_command = manager.get_program_command("devserver") + argv = shlex.split(devserver_command) + args.extra + pty.spawn(argv) diff --git a/devservices/main.py b/devservices/main.py index b244859f..1ee02ce6 100644 --- a/devservices/main.py +++ b/devservices/main.py @@ -24,6 +24,7 @@ from devservices.commands import list_services from devservices.commands import logs from devservices.commands import purge +from devservices.commands import serve from devservices.commands import status from devservices.commands import up from devservices.commands import update @@ -146,6 +147,7 @@ def main() -> None: logs.add_parser(subparsers) update.add_parser(subparsers) purge.add_parser(subparsers) + serve.add_parser(subparsers) args = parser.parse_args() diff --git a/devservices/utils/supervisor.py b/devservices/utils/supervisor.py index 579111df..e870d293 100644 --- a/devservices/utils/supervisor.py +++ b/devservices/utils/supervisor.py @@ -7,6 +7,8 @@ import subprocess import xmlrpc.client +from supervisor.options import ServerOptions + from devservices.constants import DEVSERVICES_SUPERVISOR_CONFIG_DIR from devservices.exceptions import SupervisorConfigError from devservices.exceptions import SupervisorConnectionError @@ -132,3 +134,14 @@ def stop_program(self, program_name: str) -> None: raise SupervisorProcessError( f"Failed to stop program {program_name}: {e.faultString}" ) + + def get_program_command(self, program_name: str) -> str: + opts = ServerOptions() + opts.configfile = self.config_file_path + # this reads & validates the file, populating opts.configroot + opts.process_config() + for group in opts.process_group_configs: + for proc in group.process_configs: + if proc.name == program_name and isinstance(proc.command, str): + return proc.command + raise SupervisorConfigError(f"Program {program_name} not found in config") From acb346ef5c02fe499c3b33e33bb622e0928dacc5 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 22 Apr 2025 14:13:03 -0700 Subject: [PATCH 2/4] add tests --- devservices/commands/serve.py | 24 ++++- devservices/constants.py | 1 + devservices/utils/supervisor.py | 1 - testing/utils.py | 8 ++ tests/commands/test_serve.py | 163 ++++++++++++++++++++++++++++++++ 5 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 tests/commands/test_serve.py diff --git a/devservices/commands/serve.py b/devservices/commands/serve.py index b90a8e48..06156830 100644 --- a/devservices/commands/serve.py +++ b/devservices/commands/serve.py @@ -10,7 +10,9 @@ from sentry_sdk import capture_exception from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.constants import PROCESSES_CONF_FILE_NAME from devservices.exceptions import ConfigNotFoundError +from devservices.exceptions import SupervisorConfigError from devservices.utils.console import Console from devservices.utils.services import find_matching_service from devservices.utils.supervisor import SupervisorManager @@ -34,16 +36,28 @@ def serve(args: Namespace) -> None: try: service = find_matching_service() except ConfigNotFoundError as e: - capture_exception(e, level="info") console.failure( - f"{str(e)}. Please specify a service (i.e. `devservices up sentry`) or run the command from a directory with a devservices configuration." + f"{str(e)}. Please run the command from a directory with a valid devservices configuration." ) - exit(1) + return + processes_config_path = os.path.join( + service.repo_path, f"{DEVSERVICES_DIR_NAME}/{PROCESSES_CONF_FILE_NAME}" + ) + if not os.path.exists(processes_config_path): + console.failure(f"No processes.conf file found in {processes_config_path}.") + return manager = SupervisorManager( - os.path.join(service.repo_path, f"{DEVSERVICES_DIR_NAME}/processes.conf"), + processes_config_path, service_name=service.name, ) - devserver_command = manager.get_program_command("devserver") + + try: + devserver_command = manager.get_program_command("devserver") + except SupervisorConfigError as e: + capture_exception(e, level="info") + console.failure(f"Error when getting devserver command: {str(e)}") + return + argv = shlex.split(devserver_command) + args.extra pty.spawn(argv) diff --git a/devservices/constants.py b/devservices/constants.py index 5a848eb8..00c1eac6 100644 --- a/devservices/constants.py +++ b/devservices/constants.py @@ -18,6 +18,7 @@ class Color: MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7" DEVSERVICES_DIR_NAME = "devservices" CONFIG_FILE_NAME = "config.yml" +PROCESSES_CONF_FILE_NAME = "processes.conf" DOCKER_CONFIG_DIR = os.environ.get("DOCKER_CONFIG", os.path.expanduser("~/.docker")) DOCKER_USER_PLUGIN_DIR = os.path.join(DOCKER_CONFIG_DIR, "cli-plugins/") diff --git a/devservices/utils/supervisor.py b/devservices/utils/supervisor.py index e870d293..429c902a 100644 --- a/devservices/utils/supervisor.py +++ b/devservices/utils/supervisor.py @@ -138,7 +138,6 @@ def stop_program(self, program_name: str) -> None: def get_program_command(self, program_name: str) -> str: opts = ServerOptions() opts.configfile = self.config_file_path - # this reads & validates the file, populating opts.configroot opts.process_config() for group in opts.process_group_configs: for proc in group.process_configs: diff --git a/testing/utils.py b/testing/utils.py index 9ea28fc7..921efa4b 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -26,6 +26,14 @@ def create_config_file( yaml.dump(config, f, sort_keys=False, default_flow_style=False) +def create_processes_conf_file(tmp_path: Path, config: str) -> None: + devservices_dir = Path(tmp_path, DEVSERVICES_DIR_NAME) + devservices_dir.mkdir(parents=True, exist_ok=True) + tmp_file = Path(devservices_dir, "processes.conf") + with tmp_file.open("w") as f: + f.write(config) + + def run_git_command(command: list[str], cwd: Path) -> None: subprocess.run( ["git", *command], cwd=cwd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL diff --git a/tests/commands/test_serve.py b/tests/commands/test_serve.py new file mode 100644 index 00000000..00c319f0 --- /dev/null +++ b/tests/commands/test_serve.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import os +from argparse import Namespace +from pathlib import Path +from unittest.mock import Mock +from unittest.mock import patch + +import pytest + +from devservices.commands.serve import serve +from devservices.constants import CONFIG_FILE_NAME +from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.constants import PROCESSES_CONF_FILE_NAME +from testing.utils import create_config_file +from testing.utils import create_processes_conf_file + + +@patch("devservices.commands.serve.pty.spawn") +def test_serve_success( + mock_pty_spawn: Mock, + tmp_path: Path, +) -> None: + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "redis": {"description": "Redis"}, + "clickhouse": {"description": "Clickhouse"}, + }, + "modes": {"default": ["redis", "clickhouse"]}, + }, + "services": { + "redis": {"image": "redis:6.2.14-alpine"}, + "clickhouse": { + "image": "altinity/clickhouse-server:23.8.11.29.altinitystable" + }, + }, + } + service_path = tmp_path / "example-service" + create_config_file(service_path, config) + os.chdir(service_path) + processes_config = """ +[program:devserver] +command=run devserver +autostart=true +autorestart=true +""" + create_processes_conf_file(service_path, processes_config) + + args = Namespace(extra=[]) + + serve(args) + + mock_pty_spawn.assert_called_once_with(["run", "devserver"]) + + +@patch("devservices.commands.serve.pty.spawn") +def test_serve_devservices_config_not_found( + mock_pty_spawn: Mock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "redis": {"description": "Redis"}, + "clickhouse": {"description": "Clickhouse"}, + }, + "modes": {"default": ["redis", "clickhouse"]}, + }, + "services": { + "redis": {"image": "redis:6.2.14-alpine"}, + "clickhouse": { + "image": "altinity/clickhouse-server:23.8.11.29.altinitystable" + }, + }, + } + service_path = tmp_path / "example-service" + create_config_file(service_path, config) + os.chdir(service_path) + + args = Namespace(extra=[]) + + serve(args) + + out, err = capsys.readouterr() + assert ( + out + == f"\x1b[0;31mNo processes.conf file found in {service_path}/{DEVSERVICES_DIR_NAME}/{PROCESSES_CONF_FILE_NAME}.\x1b[0m\n" + ) + mock_pty_spawn.assert_not_called() + + +@patch("devservices.commands.serve.pty.spawn") +def test_serve_processes_conf_not_found( + mock_pty_spawn: Mock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + service_path = tmp_path / "example-service" + os.makedirs(service_path) + os.chdir(service_path) + + args = Namespace(extra=[]) + + serve(args) + + out, err = capsys.readouterr() + assert ( + out + == f"\x1b[0;31mNo devservices configuration found in {service_path}/{DEVSERVICES_DIR_NAME}/{CONFIG_FILE_NAME}. Please run the command from a directory with a valid devservices configuration.\x1b[0m\n" + ) + mock_pty_spawn.assert_not_called() + + +@patch("devservices.commands.serve.pty.spawn") +def test_serve_devserver_command_not_found( + mock_pty_spawn: Mock, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + config = { + "x-sentry-service-config": { + "version": 0.1, + "service_name": "example-service", + "dependencies": { + "redis": {"description": "Redis"}, + "clickhouse": {"description": "Clickhouse"}, + }, + "modes": {"default": ["redis", "clickhouse"]}, + }, + "services": { + "redis": {"image": "redis:6.2.14-alpine"}, + "clickhouse": { + "image": "altinity/clickhouse-server:23.8.11.29.altinitystable" + }, + }, + } + service_path = tmp_path / "example-service" + create_config_file(service_path, config) + os.chdir(service_path) + processes_config = """ +[program:consumer] +command=run consumer +autostart=true +autorestart=true +""" + create_processes_conf_file(service_path, processes_config) + + args = Namespace(extra=[]) + + serve(args) + + out, err = capsys.readouterr() + assert ( + out + == "\x1b[0;31mError when getting devserver command: Program devserver not found in config\x1b[0m\n" + ) + mock_pty_spawn.assert_not_called() From 1d6b4b402ac281df7907e8021c99c86aa5a67616 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Tue, 22 Apr 2025 14:17:25 -0700 Subject: [PATCH 3/4] add more tests --- tests/utils/test_supervisor.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/utils/test_supervisor.py b/tests/utils/test_supervisor.py index 28db1863..57c96720 100644 --- a/tests/utils/test_supervisor.py +++ b/tests/utils/test_supervisor.py @@ -187,3 +187,21 @@ def test_extend_config_file( """ ) + + +def test_get_program_command_success( + supervisor_manager: SupervisorManager, tmp_path: Path +) -> None: + assert ( + supervisor_manager.get_program_command("test_program") + == "python test_program.py" + ) + + +def test_get_program_command_program_not_found( + supervisor_manager: SupervisorManager, tmp_path: Path +) -> None: + with pytest.raises( + SupervisorConfigError, match="Program nonexistent_program not found in config" + ): + supervisor_manager.get_program_command("nonexistent_program") From 8f604d9afe9f6c5d078cee8487294f0a3701c663 Mon Sep 17 00:00:00 2001 From: Hubert Deng Date: Mon, 28 Apr 2025 11:25:34 -0700 Subject: [PATCH 4/4] change terminology of program vs process --- devservices/commands/serve.py | 17 +++++++++++------ devservices/constants.py | 2 +- testing/utils.py | 5 +++-- tests/commands/test_serve.py | 16 ++++++++-------- 4 files changed, 23 insertions(+), 17 deletions(-) diff --git a/devservices/commands/serve.py b/devservices/commands/serve.py index 06156830..924a7e22 100644 --- a/devservices/commands/serve.py +++ b/devservices/commands/serve.py @@ -10,7 +10,8 @@ from sentry_sdk import capture_exception from devservices.constants import DEVSERVICES_DIR_NAME -from devservices.constants import PROCESSES_CONF_FILE_NAME +from devservices.constants import PROGRAMS_CONF_FILE_NAME +from devservices.exceptions import ConfigError from devservices.exceptions import ConfigNotFoundError from devservices.exceptions import SupervisorConfigError from devservices.utils.console import Console @@ -40,15 +41,19 @@ def serve(args: Namespace) -> None: f"{str(e)}. Please run the command from a directory with a valid devservices configuration." ) return + except ConfigError as e: + capture_exception(e) + console.failure(str(e)) + exit(1) - processes_config_path = os.path.join( - service.repo_path, f"{DEVSERVICES_DIR_NAME}/{PROCESSES_CONF_FILE_NAME}" + programs_config_path = os.path.join( + service.repo_path, f"{DEVSERVICES_DIR_NAME}/{PROGRAMS_CONF_FILE_NAME}" ) - if not os.path.exists(processes_config_path): - console.failure(f"No processes.conf file found in {processes_config_path}.") + if not os.path.exists(programs_config_path): + console.failure(f"No programs.conf file found in {programs_config_path}.") return manager = SupervisorManager( - processes_config_path, + programs_config_path, service_name=service.name, ) diff --git a/devservices/constants.py b/devservices/constants.py index 00c1eac6..5f6e5f0a 100644 --- a/devservices/constants.py +++ b/devservices/constants.py @@ -18,7 +18,7 @@ class Color: MINIMUM_DOCKER_COMPOSE_VERSION = "2.29.7" DEVSERVICES_DIR_NAME = "devservices" CONFIG_FILE_NAME = "config.yml" -PROCESSES_CONF_FILE_NAME = "processes.conf" +PROGRAMS_CONF_FILE_NAME = "programs.conf" DOCKER_CONFIG_DIR = os.environ.get("DOCKER_CONFIG", os.path.expanduser("~/.docker")) DOCKER_USER_PLUGIN_DIR = os.path.join(DOCKER_CONFIG_DIR, "cli-plugins/") diff --git a/testing/utils.py b/testing/utils.py index 921efa4b..d63b7f35 100644 --- a/testing/utils.py +++ b/testing/utils.py @@ -8,6 +8,7 @@ import yaml from devservices.constants import DEVSERVICES_DIR_NAME +from devservices.constants import PROGRAMS_CONF_FILE_NAME TESTING_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -26,10 +27,10 @@ def create_config_file( yaml.dump(config, f, sort_keys=False, default_flow_style=False) -def create_processes_conf_file(tmp_path: Path, config: str) -> None: +def create_programs_conf_file(tmp_path: Path, config: str) -> None: devservices_dir = Path(tmp_path, DEVSERVICES_DIR_NAME) devservices_dir.mkdir(parents=True, exist_ok=True) - tmp_file = Path(devservices_dir, "processes.conf") + tmp_file = Path(devservices_dir, PROGRAMS_CONF_FILE_NAME) with tmp_file.open("w") as f: f.write(config) diff --git a/tests/commands/test_serve.py b/tests/commands/test_serve.py index 00c319f0..0f4c9cf4 100644 --- a/tests/commands/test_serve.py +++ b/tests/commands/test_serve.py @@ -11,9 +11,9 @@ from devservices.commands.serve import serve from devservices.constants import CONFIG_FILE_NAME from devservices.constants import DEVSERVICES_DIR_NAME -from devservices.constants import PROCESSES_CONF_FILE_NAME +from devservices.constants import PROGRAMS_CONF_FILE_NAME from testing.utils import create_config_file -from testing.utils import create_processes_conf_file +from testing.utils import create_programs_conf_file @patch("devservices.commands.serve.pty.spawn") @@ -41,13 +41,13 @@ def test_serve_success( service_path = tmp_path / "example-service" create_config_file(service_path, config) os.chdir(service_path) - processes_config = """ + programs_config = """ [program:devserver] command=run devserver autostart=true autorestart=true """ - create_processes_conf_file(service_path, processes_config) + create_programs_conf_file(service_path, programs_config) args = Namespace(extra=[]) @@ -90,13 +90,13 @@ def test_serve_devservices_config_not_found( out, err = capsys.readouterr() assert ( out - == f"\x1b[0;31mNo processes.conf file found in {service_path}/{DEVSERVICES_DIR_NAME}/{PROCESSES_CONF_FILE_NAME}.\x1b[0m\n" + == f"\x1b[0;31mNo programs.conf file found in {service_path}/{DEVSERVICES_DIR_NAME}/{PROGRAMS_CONF_FILE_NAME}.\x1b[0m\n" ) mock_pty_spawn.assert_not_called() @patch("devservices.commands.serve.pty.spawn") -def test_serve_processes_conf_not_found( +def test_serve_programs_conf_not_found( mock_pty_spawn: Mock, tmp_path: Path, capsys: pytest.CaptureFixture[str], @@ -143,13 +143,13 @@ def test_serve_devserver_command_not_found( service_path = tmp_path / "example-service" create_config_file(service_path, config) os.chdir(service_path) - processes_config = """ + programs_config = """ [program:consumer] command=run consumer autostart=true autorestart=true """ - create_processes_conf_file(service_path, processes_config) + create_programs_conf_file(service_path, programs_config) args = Namespace(extra=[])