Refactor devchat-core for Unified Configuration File Format Support

devchat/_cli/log.py

@@ -41,7 +41,7 @@ def log(skip, max_count, topic_root, insert, delete):
     repo_chat_dir, user_chat_dir = init_dir()
 
     with handle_errors():
-        model, config = get_model_config(repo_chat_dir, user_chat_dir)
+        model, config = get_model_config(user_chat_dir)
         openai_config = OpenAIChatConfig(model=model, **config.dict(exclude_unset=True))
 
         chat = OpenAIChat(openai_config)

devchat/_cli/router.py

@@ -14,8 +14,8 @@
 def _get_model_and_config(
         model: Optional[str],
         config_str: Optional[str]):
-    repo_chat_dir, user_chat_dir = init_dir()
-    model, config = get_model_config(repo_chat_dir, user_chat_dir, model)
+    _1, user_chat_dir = init_dir()
+    model, config = get_model_config(user_chat_dir, model)
 
     parameters_data = config.dict(exclude_unset=True)
     if config_str:

devchat/_cli/topic.py

@@ -20,7 +20,7 @@ def topic(list_topics: bool, skip: int, max_count: int):
     repo_chat_dir, user_chat_dir = init_dir()
 
     with handle_errors():
-        model, config = get_model_config(repo_chat_dir, user_chat_dir)
+        model, config = get_model_config(user_chat_dir)
         parameters_data = config.dict(exclude_unset=True)
         openai_config = OpenAIChatConfig(model=model, **parameters_data)
 

devchat/_cli/utils.py

@@ -1,7 +1,6 @@
 from contextlib import contextmanager
 import os
 import sys
-import json
 import shutil
 from typing import Tuple, List, Optional, Any
 import zipfile
@@ -12,7 +11,7 @@
 except Exception:
     pass
 import rich_click as click
-from devchat.config import ConfigManager, OpenAIModelConfig
+from devchat.config import ConfigManager
 from devchat.utils import find_root_dir, add_gitignore, setup_logger, get_logger
 from devchat._cli.errors import MissContentInPromptException
 
@@ -152,37 +151,7 @@ def clone_git_repo(target_dir: str, repo_urls: List[str]):
     raise GitCommandError(f"Failed to clone repository to {target_dir}")
 
 
-def parse_legacy_config(config_json_file) -> Tuple[str, OpenAIModelConfig]:
-    with open(config_json_file, 'r', encoding='utf-8') as file:
-        legacy_data = json.load(file)
-        if 'model' not in legacy_data:
-            return None, None
-
-        model = legacy_data['model']
-        config = OpenAIModelConfig()
-        if 'tokens-per-prompt' in legacy_data:
-            config.max_input_tokens = legacy_data['tokens-per-prompt']
-        if 'OpenAI' in legacy_data:
-            if 'temperature' in legacy_data['OpenAI']:
-                config.temperature = legacy_data['OpenAI']['temperature']
-            if 'stream' in legacy_data['OpenAI']:
-                config.stream = legacy_data['OpenAI']['stream']
-        return model, config
-
-
-def get_model_config(repo_chat_dir: str, user_chat_dir: str,
+def get_model_config(user_chat_dir: str,
                      model: Optional[str] = None) -> Tuple[str, Any]:
     manager = ConfigManager(user_chat_dir)
-
-    legacy_path = os.path.join(repo_chat_dir, 'config.json')
-    if os.path.isfile(legacy_path):
-        if manager.file_is_new or os.path.getmtime(legacy_path) > manager.file_last_modified:
-            model, config = parse_legacy_config(legacy_path)
-            if model:
-                manager.config.default_model = model
-            if config:
-                manager.update_model_config(model, config)
-            manager.sync()
-            os.rename(legacy_path, legacy_path + '.old')
-
     return manager.model_config(model)

devchat/config.py

@@ -1,50 +1,18 @@
-from enum import Enum
 import os
 import sys
-from typing import List, Dict, Tuple, Union, Optional
+from typing import List, Dict, Tuple, Optional
 from pydantic import BaseModel
 import oyaml as yaml
-from devchat.openai import OpenAIChatParameters
-from devchat.anthropic import AnthropicChatParameters
 
 
-class Client(str, Enum):
-    OPENAI = "openai"
-    ANTHROPIC = "anthropic"
-    GENERAL = "general"
-
-
-class ProviderConfig(BaseModel):
-    client: Optional[str]
-
-
-class OpenAIProviderConfig(ProviderConfig, extra='forbid'):
-    api_key: Optional[str]
-    api_base: Optional[str]
-    api_type: Optional[str]
-    api_version: Optional[str]
-    deployment_name: Optional[str]
-
-
-class AnthropicProviderConfig(ProviderConfig, extra='forbid'):
+class GeneralProviderConfig(BaseModel):
     api_key: Optional[str]
     api_base: Optional[str]
-    timeout: Optional[float]
-
 
 class ModelConfig(BaseModel, extra='forbid'):
     max_input_tokens: Optional[int] = sys.maxsize
     provider: Optional[str]
 
-
-class OpenAIModelConfig(ModelConfig, OpenAIChatParameters):
-    pass
-
-
-class AnthropicModelConfig(ModelConfig, AnthropicChatParameters):
-    pass
-
-
 class GeneralModelConfig(ModelConfig):
     max_tokens: Optional[int]
     stop_sequences: Optional[List[str]]
@@ -54,11 +22,9 @@ class GeneralModelConfig(ModelConfig):
     stream: Optional[bool]
 
 
-class ChatConfig(BaseModel, extra='forbid'):
-    providers: Optional[Dict[str, Union[OpenAIProviderConfig,
-                                        AnthropicProviderConfig,
-                                        ProviderConfig]]]
-    models: Dict[str, Union[OpenAIModelConfig, AnthropicModelConfig, GeneralModelConfig]]
+class ChatConfig(BaseModel):
+    providers: Optional[Dict[str, GeneralProviderConfig]]
+    models: Dict[str, GeneralModelConfig]
     default_model: Optional[str]
 
 
@@ -86,23 +52,9 @@ def _load_and_validate_config(self) -> ChatConfig:
 
         if 'providers' in data:
             for provider, config in data['providers'].items():
-                if config['client'] == "openai":
-                    data['providers'][provider] = OpenAIProviderConfig(**config)
-                elif config['client'] == "anthropic":
-                    data['providers'][provider] = AnthropicProviderConfig(**config)
-                else:
-                    data['providers'][provider] = ProviderConfig(**config)
+                data['providers'][provider] = GeneralProviderConfig(**config)
         for model, config in data['models'].items():
-            if 'provider' not in config:
-                data['models'][model] = GeneralModelConfig(**config)
-            elif 'parameters' in config:
-                provider = data['providers'][config['provider']]
-                if provider.client == Client.OPENAI:
-                    data['models'][model] = OpenAIModelConfig(**config)
-                elif provider.client == Client.ANTHROPIC:
-                    data['models'][model] = AnthropicModelConfig(**config)
-                else:
-                    data['models'][model] = GeneralModelConfig(**config)
+            data['models'][model] = GeneralModelConfig(**config)
 
         return ChatConfig(**data)
 
@@ -120,8 +72,8 @@ def model_config(self, model_id: Optional[str] = None) -> Tuple[str, ModelConfig
     def update_model_config(
         self,
         model_id: str,
-        new_config: Union[OpenAIModelConfig, AnthropicModelConfig]
-    ) -> Union[OpenAIModelConfig, AnthropicModelConfig]:
+        new_config: GeneralModelConfig
+    ) -> GeneralModelConfig:
         _, old_config = self.model_config(model_id)
         if new_config.max_input_tokens is not None:
             old_config.max_input_tokens = new_config.max_input_tokens
@@ -137,32 +89,29 @@ def sync(self):
     def _create_sample_file(self):
         sample_config = ChatConfig(
             providers={
-                "devchat.ai": OpenAIProviderConfig(
-                    client=Client.OPENAI,
+                "devchat.ai": GeneralProviderConfig(
                     api_key=""
                 ),
-                "openai.com": OpenAIProviderConfig(
-                    client=Client.OPENAI,
+                "openai.com": GeneralProviderConfig(
                     api_key=""
                 ),
-                "general": ProviderConfig(
-                    client=Client.GENERAL
+                "general": GeneralProviderConfig(
                 )
             },
             models={
-                "gpt-4": OpenAIModelConfig(
+                "gpt-4": GeneralModelConfig(
                     max_input_tokens=6000,
                     provider='devchat.ai',
                     temperature=0,
                     stream=True
                 ),
-                "gpt-3.5-turbo-16k": OpenAIModelConfig(
+                "gpt-3.5-turbo-16k": GeneralModelConfig(
                     max_input_tokens=12000,
                     provider='devchat.ai',
                     temperature=0,
                     stream=True
                 ),
-                "gpt-3.5-turbo": OpenAIModelConfig(
+                "gpt-3.5-turbo": GeneralModelConfig(
                     max_input_tokens=3000,
                     provider='devchat.ai',
                     temperature=0,

tests/test_cli_prompt.py

@@ -2,7 +2,7 @@
 import json
 import pytest
 from click.testing import CliRunner
-from devchat.config import ConfigManager, OpenAIModelConfig
+from devchat.config import ConfigManager, GeneralModelConfig
 from devchat._cli.main import main
 from devchat.utils import openai_response_tokens
 from devchat.utils import check_format, get_content, get_prompt_hash
@@ -198,7 +198,7 @@ def test_prompt_without_repo(mock_home_dir):  # pylint: disable=W0613
 def test_prompt_tokens_exceed_config(mock_home_dir):  # pylint: disable=W0613
     model = "gpt-3.5-turbo"
     max_input_tokens = 2000
-    config = OpenAIModelConfig(
+    config = GeneralModelConfig(
         max_input_tokens=max_input_tokens,
         temperature=0
     )
@@ -222,7 +222,7 @@ def test_prompt_tokens_exceed_config(mock_home_dir):  # pylint: disable=W0613
 def test_file_tokens_exceed_config(mock_home_dir, tmpdir):  # pylint: disable=W0613
     model = "gpt-3.5-turbo"
     max_input_tokens = 2000
-    config = OpenAIModelConfig(
+    config = GeneralModelConfig(
         max_input_tokens=max_input_tokens,
         temperature=0
     )

tests/test_config.py

@@ -1,7 +1,6 @@
 import os
 from click.testing import CliRunner
-from devchat.config import ConfigManager, OpenAIModelConfig, ChatConfig
-from devchat._cli.main import main
+from devchat.config import ConfigManager, GeneralModelConfig, ChatConfig
 
 runner = CliRunner()
 
@@ -27,7 +26,7 @@ def test_get_model_config(tmp_path):
 def test_update_model_config(tmp_path):
     model = 'gpt-4'
     config_manager = ConfigManager(tmp_path)
-    config = OpenAIModelConfig(
+    config = GeneralModelConfig(
         max_input_tokens=7000,
         temperature=0.5
     )
@@ -40,55 +39,3 @@ def test_update_model_config(tmp_path):
     config_manager.sync()
     fresh_config_manager = ConfigManager(tmp_path)
     assert config_manager.config == fresh_config_manager.config
-
-
-def test_legacy_config(git_repo, mock_home_dir):
-    repo_chat_dir = os.path.join(git_repo, ".chat")
-    os.makedirs(repo_chat_dir)
-    home_chat_dir = os.path.join(mock_home_dir, ".chat")
-
-    legacy_config = '''
-    {
-        "model": "gpt-3.5-turbo",
-        "provider": "OpenAI",
-        "tokens-per-prompt": 2000,
-        "OpenAI": {
-            "temperature": 0.2
-        }
-    }
-    '''
-
-    legacy_file = os.path.join(repo_chat_dir, "config.json")
-    with open(legacy_file, 'w', encoding='utf-8') as file:
-        file.write(legacy_config)
-
-    result = runner.invoke(main, ['log'])
-    assert result.exit_code == 0
-    assert not os.path.exists(legacy_file)
-    assert os.path.isfile(legacy_file + '.old')
-    config_manager = ConfigManager(home_chat_dir)
-    assert config_manager.config.default_model == 'gpt-3.5-turbo'
-    assert config_manager.config.models['gpt-3.5-turbo'].max_input_tokens == 2000
-    assert config_manager.config.models['gpt-3.5-turbo'].temperature == 0.2
-    assert config_manager.config.models['gpt-3.5-turbo'].stream is True
-
-    legacy_config = '''
-    {
-        "model": "gpt-3.5-turbo",
-        "provider": "OpenAI",
-        "tokens-per-prompt": 2000,
-        "OpenAI": {
-            "stream": false
-        }
-    }
-    '''
-    with open(legacy_file, 'w', encoding='utf-8') as file:
-        file.write(legacy_config)
-
-    result = runner.invoke(main, ['prompt', "This is a test. Reply nothing."])
-    assert result.exit_code == 0
-    config_manager = ConfigManager(home_chat_dir)
-    assert config_manager.config.default_model == 'gpt-3.5-turbo'
-    assert config_manager.config.models['gpt-3.5-turbo'].max_input_tokens == 2000
-    assert config_manager.config.models['gpt-3.5-turbo'].temperature == 0.2
-    assert config_manager.config.models['gpt-3.5-turbo'].stream is False