diff --git a/.circleci/config.yml b/.circleci/config.yml index c047736a..90ebbdbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,9 +15,9 @@ jobs: command: | poetry install - run: - name: Run Pylint + name: Run linter command: | - poetry run pylint devchat tests + make check - run: name: Run Pytest command: | diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 7acd8478..00000000 --- a/.flake8 +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 100 -ignore = E712,W503 \ No newline at end of file diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml new file mode 100644 index 00000000..01ae2cb5 --- /dev/null +++ b/.github/workflows/dev.yaml @@ -0,0 +1,31 @@ +name: Dev + +on: + pull_request: + branches: [main] + push: + branches: [main] + +jobs: + lint-and-test: + name: Run linter and test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + - uses: abatilo/actions-poetry@v2 + - name: Install Python dependencies + run: | + poetry install + + - name: Run linter + run: | + make check + + - name: Run Pytest + run: | + poetry run pytest + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_FOR_TEST }} diff --git a/.gitmodules b/.gitmodules index 0d409a52..e69de29b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "data/workflows"] - path = data/workflows - url = https://github.com/devchat-ai/workflows.git diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 84bb93b9..00000000 --- a/.pylintrc +++ /dev/null @@ -1,17 +0,0 @@ -[BASIC] -good-names=id,db - -[MESSAGES CONTROL] -disable= - missing-docstring, - broad-exception-caught, - too-few-public-methods, - too-many-instance-attributes, - too-many-arguments, - too-many-locals, - too-many-branches, - fixme, - R0801 - -[MASTER] -extension-pkg-whitelist=pydantic diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c9e33c47 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +.PHONY: check fix + +div = $(shell printf '=%.0s' {1..120}) + +DIR="." +check: + @echo ${div} + poetry run ruff check $(DIR) + poetry run ruff format $(DIR) --check + @echo "Done!" + +fix: + @echo ${div} + poetry run ruff format $(DIR) + @echo ${div} + poetry run ruff check $(DIR) --fix + @echo "Done!" diff --git a/data/workflows b/data/workflows deleted file mode 160000 index 32ebbc93..00000000 --- a/data/workflows +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 32ebbc934bec4b8a4ee0596a71eed45a2f98bdb6 diff --git a/devchat/_cli/__init__.py b/devchat/_cli/__init__.py index 29b351dd..8c291d58 100644 --- a/devchat/_cli/__init__.py +++ b/devchat/_cli/__init__.py @@ -1,17 +1,18 @@ import os + from .log import log from .prompt import prompt +from .route import route from .run import run from .topic import topic -from .route import route script_dir = os.path.dirname(os.path.realpath(__file__)) -os.environ['TIKTOKEN_CACHE_DIR'] = os.path.join(script_dir, '..', 'tiktoken_cache') +os.environ["TIKTOKEN_CACHE_DIR"] = os.path.join(script_dir, "..", "tiktoken_cache") __all__ = [ - 'log', - 'prompt', - 'run', - 'topic', - 'route', + "log", + "prompt", + "run", + "topic", + "route", ] diff --git a/devchat/_cli/errors.py b/devchat/_cli/errors.py index 73b89920..3becb31b 100644 --- a/devchat/_cli/errors.py +++ b/devchat/_cli/errors.py @@ -1,4 +1,2 @@ - - class MissContentInPromptException(Exception): pass diff --git a/devchat/_cli/log.py b/devchat/_cli/log.py index ac476311..70277d5a 100755 --- a/devchat/_cli/log.py +++ b/devchat/_cli/log.py @@ -1,13 +1,12 @@ -# pylint: disable=import-outside-toplevel - import json import sys import time -from typing import Optional, List, Dict from dataclasses import dataclass, field +from typing import Dict, List, Optional import click + @dataclass class PromptData: model: str = "none" @@ -19,28 +18,34 @@ class PromptData: response_tokens: int = 0 -@click.command(help='Process logs') -@click.option('--skip', default=0, help='Skip number prompts before showing the prompt history.') -@click.option('-n', '--max-count', default=1, help='Limit the number of commits to output.') -@click.option('-t', '--topic', 'topic_root', default=None, - help='Hash of the root prompt of the topic to select prompts from.') -@click.option('--insert', default=None, help='JSON string of the prompt to insert into the log.') -@click.option('--delete', default=None, help='Hash of the leaf prompt to delete from the log.') +@click.command(help="Process logs") +@click.option("--skip", default=0, help="Skip number prompts before showing the prompt history.") +@click.option("-n", "--max-count", default=1, help="Limit the number of commits to output.") +@click.option( + "-t", + "--topic", + "topic_root", + default=None, + help="Hash of the root prompt of the topic to select prompts from.", +) +@click.option("--insert", default=None, help="JSON string of the prompt to insert into the log.") +@click.option("--delete", default=None, help="Hash of the leaf prompt to delete from the log.") def log(skip, max_count, topic_root, insert, delete): """ Manage the prompt history. """ + from devchat._cli.utils import get_model_config, handle_errors, init_dir from devchat.openai.openai_chat import OpenAIChat, OpenAIChatConfig, OpenAIPrompt - from devchat.store import Store - from devchat._cli.utils import handle_errors, init_dir, get_model_config from devchat.utils import get_logger, get_user_info logger = get_logger(__name__) if (insert or delete) and (skip != 0 or max_count != 1 or topic_root is not None): - print("Error: The --insert or --delete option cannot be used with other options.", - file=sys.stderr) + print( + "Error: The --insert or --delete option cannot be used with other options.", + file=sys.stderr, + ) sys.exit(1) repo_chat_dir, user_chat_dir = init_dir() diff --git a/devchat/_cli/main.py b/devchat/_cli/main.py index f1672a65..0247b144 100755 --- a/devchat/_cli/main.py +++ b/devchat/_cli/main.py @@ -1,15 +1,11 @@ """ This module contains the main function for the DevChat CLI. """ + import click +from devchat._cli import log, prompt, route, run, topic from devchat.utils import get_logger -from devchat._cli import log -from devchat._cli import prompt -from devchat._cli import run -from devchat._cli import topic -from devchat._cli import route - from devchat.workflow.cli import workflow logger = get_logger(__name__) diff --git a/devchat/_cli/prompt.py b/devchat/_cli/prompt.py index 73a2aa17..1dd23268 100755 --- a/devchat/_cli/prompt.py +++ b/devchat/_cli/prompt.py @@ -1,33 +1,61 @@ -# pylint: disable=import-outside-toplevel import sys from typing import List, Optional import click -@click.command(help='Interact with the large language model (LLM).') -@click.argument('content', required=False) -@click.option('-p', '--parent', help='Input the parent prompt hash to continue the conversation.') -@click.option('-r', '--reference', multiple=True, - help='Input one or more specific previous prompts to include in the current prompt.') -@click.option('-i', '--instruct', multiple=True, - help='Add one or more files to the prompt as instructions.') -@click.option('-c', '--context', multiple=True, - help='Add one or more files to the prompt as a context.') -@click.option('-m', '--model', help='Specify the model to use for the prompt.') -@click.option('--config', 'config_str', - help='Specify a JSON string to overwrite the default configuration for this prompt.') -@click.option('-f', '--functions', type=click.Path(exists=True), - help='Path to a JSON file with functions for the prompt.') -@click.option('-n', '--function-name', - help='Specify the function name when the content is the output of a function.') -@click.option('-ns', '--not-store', is_flag=True, default=False, required=False, - help='Do not save the conversation to the store.') -def prompt(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None, - functions: Optional[str] = None, function_name: Optional[str] = None, - not_store: Optional[bool] = False): +@click.command(help="Interact with the large language model (LLM).") +@click.argument("content", required=False) +@click.option("-p", "--parent", help="Input the parent prompt hash to continue the conversation.") +@click.option( + "-r", + "--reference", + multiple=True, + help="Input one or more specific previous prompts to include in the current prompt.", +) +@click.option( + "-i", "--instruct", multiple=True, help="Add one or more files to the prompt as instructions." +) +@click.option( + "-c", "--context", multiple=True, help="Add one or more files to the prompt as a context." +) +@click.option("-m", "--model", help="Specify the model to use for the prompt.") +@click.option( + "--config", + "config_str", + help="Specify a JSON string to overwrite the default configuration for this prompt.", +) +@click.option( + "-f", + "--functions", + type=click.Path(exists=True), + help="Path to a JSON file with functions for the prompt.", +) +@click.option( + "-n", + "--function-name", + help="Specify the function name when the content is the output of a function.", +) +@click.option( + "-ns", + "--not-store", + is_flag=True, + default=False, + required=False, + help="Do not save the conversation to the store.", +) +def prompt( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, + functions: Optional[str] = None, + function_name: Optional[str] = None, + not_store: Optional[bool] = False, +): """ This command performs interactions with the specified large language model (LLM) by sending prompts and receiving responses. @@ -62,6 +90,7 @@ def prompt(content: Optional[str], parent: Optional[str], reference: Optional[Li """ from devchat._cli.router import llm_prompt + llm_prompt( content, parent, @@ -72,5 +101,6 @@ def prompt(content: Optional[str], parent: Optional[str], reference: Optional[Li config_str, functions, function_name, - not_store) + not_store, + ) sys.exit(0) diff --git a/devchat/_cli/route.py b/devchat/_cli/route.py index 106e0bcc..4d7bc449 100755 --- a/devchat/_cli/route.py +++ b/devchat/_cli/route.py @@ -1,28 +1,48 @@ -# pylint: disable=import-outside-toplevel import sys from typing import List, Optional import click -@click.command(help='Route a prompt to the specified LLM') -@click.argument('content', required=False) -@click.option('-p', '--parent', help='Input the parent prompt hash to continue the conversation.') -@click.option('-r', '--reference', multiple=True, - help='Input one or more specific previous prompts to include in the current prompt.') -@click.option('-i', '--instruct', multiple=True, - help='Add one or more files to the prompt as instructions.') -@click.option('-c', '--context', multiple=True, - help='Add one or more files to the prompt as a context.') -@click.option('-m', '--model', help='Specify the model to use for the prompt.') -@click.option('--config', 'config_str', - help='Specify a JSON string to overwrite the default configuration for this prompt.') -@click.option('-a', '--auto', is_flag=True, default=False, required=False, - help='Answer question by function-calling.') -def route(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None, - auto: Optional[bool] = False): +@click.command(help="Route a prompt to the specified LLM") +@click.argument("content", required=False) +@click.option("-p", "--parent", help="Input the parent prompt hash to continue the conversation.") +@click.option( + "-r", + "--reference", + multiple=True, + help="Input one or more specific previous prompts to include in the current prompt.", +) +@click.option( + "-i", "--instruct", multiple=True, help="Add one or more files to the prompt as instructions." +) +@click.option( + "-c", "--context", multiple=True, help="Add one or more files to the prompt as a context." +) +@click.option("-m", "--model", help="Specify the model to use for the prompt.") +@click.option( + "--config", + "config_str", + help="Specify a JSON string to overwrite the default configuration for this prompt.", +) +@click.option( + "-a", + "--auto", + is_flag=True, + default=False, + required=False, + help="Answer question by function-calling.", +) +def route( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, + auto: Optional[bool] = False, +): """ This command performs interactions with the specified large language model (LLM) by sending prompts and receiving responses. @@ -58,14 +78,5 @@ def route(content: Optional[str], parent: Optional[str], reference: Optional[Lis """ from devchat._cli.router import llm_route - llm_route( - content, - parent, - reference, - instruct, - context, - model, - config_str, - auto - ) + llm_route(content, parent, reference, instruct, context, model, config_str, auto) sys.exit(0) diff --git a/devchat/_cli/router.py b/devchat/_cli/router.py index 00e7f2c8..3446757b 100755 --- a/devchat/_cli/router.py +++ b/devchat/_cli/router.py @@ -1,4 +1,3 @@ -# pylint: disable=import-outside-toplevel import json import sys from typing import List, Optional @@ -6,10 +5,8 @@ from devchat.workflow.workflow import Workflow -def _get_model_and_config( - model: Optional[str], - config_str: Optional[str]): - from devchat._cli.utils import init_dir, get_model_config +def _get_model_and_config(model: Optional[str], config_str: Optional[str]): + from devchat._cli.utils import get_model_config, init_dir _1, user_chat_dir = init_dir() model, config = get_model_config(user_chat_dir, model) @@ -20,15 +17,17 @@ def _get_model_and_config( parameters_data.update(config_data) return model, parameters_data + def _load_tool_functions(functions: Optional[str]): try: if functions: - with open(functions, 'r', encoding="utf-8") as f_file: + with open(functions, "r", encoding="utf-8") as f_file: return json.load(f_file) return None except Exception: return None + def _load_instruction_contents(content: str, instruct: Optional[List[str]]): from devchat.engine import load_workflow_instruction from devchat.utils import parse_files @@ -41,24 +40,31 @@ def _load_instruction_contents(content: str, instruct: Optional[List[str]]): return instruct_contents -def before_prompt(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None, - functions: Optional[str] = None, function_name: Optional[str] = None, - not_store: Optional[bool] = False): +def before_prompt( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, + functions: Optional[str] = None, + function_name: Optional[str] = None, + not_store: Optional[bool] = False, +): + from devchat._cli.errors import MissContentInPromptException + from devchat._cli.utils import init_dir from devchat.assistant import Assistant from devchat.openai.openai_chat import OpenAIChat, OpenAIChatConfig from devchat.store import Store from devchat.utils import parse_files - from devchat._cli.utils import init_dir - from devchat._cli.errors import MissContentInPromptException repo_chat_dir, _1 = init_dir() if content is None: content = sys.stdin.read() - if content == '': + if content == "": raise MissContentInPromptException() instruct_contents = _load_instruction_contents(content, instruct) @@ -74,53 +80,80 @@ def before_prompt(content: Optional[str], parent: Optional[str], reference: Opti assistant = Assistant(chat, chat_store, max_input_tokens, not not_store) assistant.make_prompt( - request = content, - instruct_contents = instruct_contents, - context_contents = context_contents, - functions = tool_functions, + request=content, + instruct_contents=instruct_contents, + context_contents=context_contents, + functions=tool_functions, parent=parent, references=reference, - function_name=function_name + function_name=function_name, ) return model, assistant, content -def llm_prompt(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None, - functions: Optional[str] = None, function_name: Optional[str] = None, - not_store: Optional[bool] = False): + +def llm_prompt( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, + functions: Optional[str] = None, + function_name: Optional[str] = None, + not_store: Optional[bool] = False, +): from devchat._cli.utils import handle_errors with handle_errors(): - _1, assistant, _3, = before_prompt( - content, parent, reference, instruct, context, - model, config_str, functions, function_name, not_store - ) + ( + _1, + assistant, + _3, + ) = before_prompt( + content, + parent, + reference, + instruct, + context, + model, + config_str, + functions, + function_name, + not_store, + ) print(assistant.prompt.formatted_header()) for response in assistant.iterate_response(): - print(response, end='', flush=True) - - -def llm_commmand(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None): - from devchat.engine import run_command + print(response, end="", flush=True) + + +def llm_commmand( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, +): from devchat._cli.utils import handle_errors + from devchat.engine import run_command with handle_errors(): model, assistant, content = before_prompt( content, parent, reference, instruct, context, model, config_str, None, None, True - ) + ) print(assistant.prompt.formatted_header()) command_result = run_command( - model_name = model, - history_messages = assistant.prompt.messages, - input_text = content, - parent_hash = parent, - auto_fun = False) + model_name=model, + history_messages=assistant.prompt.messages, + input_text=content, + parent_hash=parent, + auto_fun=False, + ) if command_result is not None: sys.exit(0) @@ -129,17 +162,23 @@ def llm_commmand(content: Optional[str], parent: Optional[str], reference: Optio sys.exit(-1) -def llm_route(content: Optional[str], parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None, - auto: Optional[bool] = False): - from devchat.engine import run_command +def llm_route( + content: Optional[str], + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, + auto: Optional[bool] = False, +): from devchat._cli.utils import handle_errors + from devchat.engine import run_command with handle_errors(): model, assistant, content = before_prompt( content, parent, reference, instruct, context, model, config_str, None, None, True - ) + ) name, user_input = Workflow.parse_trigger(content) workflow = Workflow.load(name) if name else None @@ -165,13 +204,14 @@ def llm_route(content: Optional[str], parent: Optional[str], reference: Optional print(assistant.prompt.formatted_header()) command_result = run_command( - model_name = model, - history_messages = assistant.prompt.messages, - input_text = content, - parent_hash = parent, - auto_fun = auto) + model_name=model, + history_messages=assistant.prompt.messages, + input_text=content, + parent_hash=parent, + auto_fun=auto, + ) if command_result is not None: sys.exit(command_result[0]) for response in assistant.iterate_response(): - print(response, end='', flush=True) + print(response, end="", flush=True) diff --git a/devchat/_cli/run.py b/devchat/_cli/run.py index ab028413..f4d8054d 100755 --- a/devchat/_cli/run.py +++ b/devchat/_cli/run.py @@ -1,49 +1,82 @@ -# pylint: disable=import-outside-toplevel from typing import List, Optional, Tuple import click + @click.command( - help="The 'command' argument is the name of the command to run or get information about.") -@click.argument('command', required=False, default='') -@click.option('--list', 'list_flag', is_flag=True, default=False, - help='List all specified commands in JSON format.') -@click.option('--recursive', '-r', 'recursive_flag', is_flag=True, default=True, - help='List commands recursively.') -@click.option('--update-sys', 'update_sys_flag', is_flag=True, default=False, - help='Pull the `sys` command directory from the DevChat repository.') -@click.option('-p', '--parent', help='Input the parent prompt hash to continue the conversation.') -@click.option('-r', '--reference', multiple=True, - help='Input one or more specific previous prompts to include in the current prompt.') -@click.option('-i', '--instruct', multiple=True, - help='Add one or more files to the prompt as instructions.') -@click.option('-c', '--context', multiple=True, - help='Add one or more files to the prompt as a context.') -@click.option('-m', '--model', help='Specify the model to use for the prompt.') -@click.option('--config', 'config_str', - help='Specify a JSON string to overwrite the default configuration for this prompt.') -# pylint: disable=redefined-outer-name -def run(command: str, list_flag: bool, recursive_flag: bool, update_sys_flag: bool, - parent: Optional[str], reference: Optional[List[str]], - instruct: Optional[List[str]], context: Optional[List[str]], - model: Optional[str], config_str: Optional[str] = None): + help="The 'command' argument is the name of the command to run or get information about." +) +@click.argument("command", required=False, default="") +@click.option( + "--list", + "list_flag", + is_flag=True, + default=False, + help="List all specified commands in JSON format.", +) +@click.option( + "--recursive", + "-r", + "recursive_flag", + is_flag=True, + default=True, + help="List commands recursively.", +) +@click.option( + "--update-sys", + "update_sys_flag", + is_flag=True, + default=False, + help="Pull the `sys` command directory from the DevChat repository.", +) +@click.option("-p", "--parent", help="Input the parent prompt hash to continue the conversation.") +@click.option( + "-r", + "--reference", + multiple=True, + help="Input one or more specific previous prompts to include in the current prompt.", +) +@click.option( + "-i", "--instruct", multiple=True, help="Add one or more files to the prompt as instructions." +) +@click.option( + "-c", "--context", multiple=True, help="Add one or more files to the prompt as a context." +) +@click.option("-m", "--model", help="Specify the model to use for the prompt.") +@click.option( + "--config", + "config_str", + help="Specify a JSON string to overwrite the default configuration for this prompt.", +) +def run( + command: str, + list_flag: bool, + recursive_flag: bool, + update_sys_flag: bool, + parent: Optional[str], + reference: Optional[List[str]], + instruct: Optional[List[str]], + context: Optional[List[str]], + model: Optional[str], + config_str: Optional[str] = None, +): """ Operate the workflow engine of DevChat. """ import json import os import sys - from devchat._cli.utils import init_dir, handle_errors - from devchat.engine import Namespace, CommandParser - from devchat.utils import get_logger from devchat._cli.router import llm_commmand + from devchat._cli.utils import handle_errors, init_dir + from devchat.engine import CommandParser, Namespace + from devchat.utils import get_logger logger = get_logger(__name__) _, user_chat_dir = init_dir() with handle_errors(): - workflows_dir = os.path.join(user_chat_dir, 'workflows') + workflows_dir = os.path.join(user_chat_dir, "workflows") if not os.path.exists(workflows_dir): os.makedirs(workflows_dir) if not os.path.isdir(workflows_dir): @@ -54,14 +87,14 @@ def run(command: str, list_flag: bool, recursive_flag: bool, update_sys_flag: bo commander = CommandParser(namespace) if update_sys_flag: - sys_dir = os.path.join(workflows_dir, 'sys') + sys_dir = os.path.join(workflows_dir, "sys") git_urls = [ - ('https://gitlab.com/devchat-ai/workflows.git', 'main'), - ('https://github.com/devchat-ai/workflows.git', 'main') + ("https://gitlab.com/devchat-ai/workflows.git", "main"), + ("https://github.com/devchat-ai/workflows.git", "main"), ] zip_urls = [ - 'https://gitlab.com/devchat-ai/workflows/-/archive/main/workflows-main.zip', - 'https://codeload.github.com/devchat-ai/workflows/zip/refs/heads/main' + "https://gitlab.com/devchat-ai/workflows/-/archive/main/workflows-main.zip", + "https://codeload.github.com/devchat-ai/workflows/zip/refs/heads/main", ] _clone_or_pull_git_repo(sys_dir, git_urls, zip_urls) return @@ -73,26 +106,15 @@ def run(command: str, list_flag: bool, recursive_flag: bool, update_sys_flag: bo if not cmd: logger.warning("Existing command directory failed to parse: %s", name) continue - commands.append({ - 'name': name, - 'description': cmd.description, - 'path': cmd.path - }) + commands.append({"name": name, "description": cmd.description, "path": cmd.path}) print(json.dumps(commands, indent=2)) return if command: - llm_commmand( - command, - parent, - reference, - instruct, - context, - model, - config_str - ) + llm_commmand(command, parent, reference, instruct, context, model, config_str) return + def __onerror(func, path, _1): """ Error handler for shutil.rmtree. @@ -114,18 +136,21 @@ def __onerror(func, path, _1): # Retry the function that failed func(path) + def __make_files_writable(directory): """ Recursively make all files in the directory writable. """ import os import stat + for root, _1, files in os.walk(directory): for name in files: filepath = os.path.join(root, name) if not os.access(filepath, os.W_OK): os.chmod(filepath, stat.S_IWUSR) + def _clone_or_pull_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]], zip_urls: List[str]): """ Clone a Git repository to a specified location, or pull it if it already exists. @@ -135,13 +160,13 @@ def _clone_or_pull_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]], z """ import os import shutil + + from devchat._cli.utils import clone_git_repo, download_and_extract_workflow from devchat.utils import get_logger - from devchat._cli.utils import download_and_extract_workflow - from devchat._cli.utils import clone_git_repo logger = get_logger(__name__) - if shutil.which('git') is None: + if shutil.which("git") is None: # If Git is not installed, download and extract the workflow for url in zip_urls: try: @@ -152,13 +177,13 @@ def _clone_or_pull_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]], z return if os.path.exists(target_dir): - bak_dir = target_dir + '_bak' - new_dir = target_dir + '_old' + bak_dir = target_dir + "_bak" + new_dir = target_dir + "_old" if os.path.exists(new_dir): shutil.rmtree(new_dir, onerror=__onerror) if os.path.exists(bak_dir): shutil.rmtree(bak_dir, onerror=__onerror) - print(f'{target_dir} is already exists. Moved to {new_dir}') + print(f"{target_dir} is already exists. Moved to {new_dir}") clone_git_repo(bak_dir, repo_urls) try: shutil.move(target_dir, new_dir) @@ -173,4 +198,4 @@ def _clone_or_pull_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]], z else: clone_git_repo(target_dir, repo_urls) - print(f'Updated {target_dir}') + print(f"Updated {target_dir}") diff --git a/devchat/_cli/topic.py b/devchat/_cli/topic.py index 63c90e2c..d68885bf 100755 --- a/devchat/_cli/topic.py +++ b/devchat/_cli/topic.py @@ -1,19 +1,21 @@ -# pylint: disable=import-outside-toplevel import click -@click.command(help='Manage topics') -@click.option('--list', '-l', 'list_topics', is_flag=True, - help='List topics in reverse chronological order.') -@click.option('--skip', default=0, help='Skip number of topics before showing the list.') -@click.option('-n', '--max-count', default=100, help='Limit the number of topics to output.') + +@click.command(help="Manage topics") +@click.option( + "--list", "-l", "list_topics", is_flag=True, help="List topics in reverse chronological order." +) +@click.option("--skip", default=0, help="Skip number of topics before showing the list.") +@click.option("-n", "--max-count", default=100, help="Limit the number of topics to output.") def topic(list_topics: bool, skip: int, max_count: int): """ Manage topics. """ import json + + from devchat._cli.utils import get_model_config, handle_errors, init_dir + from devchat.openai import OpenAIChat, OpenAIChatConfig from devchat.store import Store - from devchat.openai import OpenAIChatConfig, OpenAIChat - from devchat._cli.utils import init_dir, handle_errors, get_model_config repo_chat_dir, user_chat_dir = init_dir() diff --git a/devchat/_cli/utils.py b/devchat/_cli/utils.py index 22369c5a..d503afe4 100755 --- a/devchat/_cli/utils.py +++ b/devchat/_cli/utils.py @@ -1,32 +1,32 @@ -# pylint: disable=import-outside-toplevel -from contextlib import contextmanager import os -import sys import shutil -from typing import Tuple, List, Optional, Any +import sys import zipfile +from contextlib import contextmanager +from typing import Any, List, Optional, Tuple from devchat._cli.errors import MissContentInPromptException -from devchat.utils import find_root_dir, add_gitignore, setup_logger, get_logger +from devchat.utils import add_gitignore, find_root_dir, get_logger, setup_logger logger = get_logger(__name__) def download_and_extract_workflow(workflow_url, target_dir): import requests + # Download the workflow zip file response = requests.get(workflow_url, stream=True, timeout=10) # Downaload file to temp dir os.makedirs(target_dir, exist_ok=True) - zip_path = os.path.join(target_dir, 'workflow.zip') - with open(zip_path, 'wb') as file_handle: + zip_path = os.path.join(target_dir, "workflow.zip") + with open(zip_path, "wb") as file_handle: for chunk in response.iter_content(chunk_size=8192): if chunk: file_handle.write(chunk) # Extract the zip file parent_dir = os.path.dirname(target_dir) - with zipfile.ZipFile(zip_path, 'r') as zip_ref: + with zipfile.ZipFile(zip_path, "r") as zip_ref: zip_ref.extractall(parent_dir) # Delete target directory if exists @@ -34,7 +34,7 @@ def download_and_extract_workflow(workflow_url, target_dir): shutil.rmtree(target_dir) # Rename extracted directory to target directory - extracted_dir = os.path.join(parent_dir, 'workflows-main') + extracted_dir = os.path.join(parent_dir, "workflows-main") os.rename(extracted_dir, target_dir) @@ -58,9 +58,11 @@ def handle_errors(): print(f"{type(error).__name__}: {error}", file=sys.stderr) sys.exit(1) + REPO_CHAT_DIR = None USER_CHAT_DIR = None + def init_dir() -> Tuple[str, str]: """ Initialize the chat directories. @@ -69,7 +71,6 @@ def init_dir() -> Tuple[str, str]: REPO_CHAT_DIR: The chat directory in the repository. USER_CHAT_DIR: The chat directory in the user's home. """ - # pylint: disable=global-statement global REPO_CHAT_DIR global USER_CHAT_DIR if REPO_CHAT_DIR and USER_CHAT_DIR: @@ -108,8 +109,8 @@ def init_dir() -> Tuple[str, str]: sys.exit(1) try: - setup_logger(os.path.join(REPO_CHAT_DIR, 'error.log')) - add_gitignore(REPO_CHAT_DIR, '*') + setup_logger(os.path.join(REPO_CHAT_DIR, "error.log")) + add_gitignore(REPO_CHAT_DIR, "*") except Exception as exc: logger.error("Failed to setup logger or add .gitignore: %s", exc) @@ -125,7 +126,7 @@ def valid_git_repo(target_dir: str, valid_urls: List[str]) -> bool: :return: True if the directory is a valid Git repository with a valid URL, False otherwise. """ try: - from git import Repo, InvalidGitRepositoryError + from git import InvalidGitRepositoryError, Repo except Exception: pass @@ -148,7 +149,7 @@ def clone_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]]): :param repo_urls: A list of possible Git repository URLs. """ try: - from git import Repo, GitCommandError + from git import GitCommandError, Repo except Exception: pass @@ -164,8 +165,8 @@ def clone_git_repo(target_dir: str, repo_urls: List[Tuple[str, str]]): raise GitCommandError(f"Failed to clone repository to {target_dir}") -def get_model_config(user_chat_dir: str, - model: Optional[str] = None) -> Tuple[str, Any]: +def get_model_config(user_chat_dir: str, model: Optional[str] = None) -> Tuple[str, Any]: from devchat.config import ConfigManager + manager = ConfigManager(user_chat_dir) return manager.model_config(model) diff --git a/devchat/anthropic/__init__.py b/devchat/anthropic/__init__.py index f951c956..ee48977a 100644 --- a/devchat/anthropic/__init__.py +++ b/devchat/anthropic/__init__.py @@ -1,5 +1,3 @@ from .anthropic_chat import AnthropicChatParameters -__all__ = [ - 'AnthropicChatParameters' -] +__all__ = ["AnthropicChatParameters"] diff --git a/devchat/anthropic/anthropic_chat.py b/devchat/anthropic/anthropic_chat.py index 6cf72022..93b1b796 100644 --- a/devchat/anthropic/anthropic_chat.py +++ b/devchat/anthropic/anthropic_chat.py @@ -1,8 +1,9 @@ -from typing import List, Optional, Dict, Any +from typing import Any, Dict, List, Optional + from pydantic import BaseModel, Field -class AnthropicChatParameters(BaseModel, extra='ignore'): +class AnthropicChatParameters(BaseModel, extra="ignore"): max_tokens_to_sample: int = Field(1024, ge=1) stop_sequences: Optional[List[str]] temperature: Optional[float] = Field(0.2, ge=0, le=1) diff --git a/devchat/assistant.py b/devchat/assistant.py index ec20e55a..fd76f2c8 100755 --- a/devchat/assistant.py +++ b/devchat/assistant.py @@ -1,15 +1,14 @@ import json import sys import time -from typing import Optional, List, Iterator +from typing import Iterator, List, Optional -from devchat.message import Message from devchat.chat import Chat +from devchat.message import Message from devchat.openai.openai_prompt import OpenAIPrompt from devchat.store import Store from devchat.utils import get_logger - logger = get_logger(__name__) @@ -37,14 +36,20 @@ def available_tokens(self) -> int: def _check_limit(self): if self._prompt.request_tokens > self.token_limit: - raise ValueError(f"Prompt tokens {self._prompt.request_tokens} " - f"beyond limit {self.token_limit}.") - - def make_prompt(self, request: str, - instruct_contents: Optional[List[str]], context_contents: Optional[List[str]], - functions: Optional[List[dict]], - parent: Optional[str] = None, references: Optional[List[str]] = None, - function_name: Optional[str] = None): + raise ValueError( + f"Prompt tokens {self._prompt.request_tokens} " f"beyond limit {self.token_limit}." + ) + + def make_prompt( + self, + request: str, + instruct_contents: Optional[List[str]], + context_contents: Optional[List[str]], + functions: Optional[List[dict]], + parent: Optional[str] = None, + references: Optional[List[str]] = None, + function_name: Optional[str] = None, + ): """ Make a prompt for the chat API. @@ -59,7 +64,7 @@ def make_prompt(self, request: str, self._check_limit() # Add instructions to the prompt if instruct_contents: - combined_instruct = ''.join(instruct_contents) + combined_instruct = "".join(instruct_contents) self._prompt.append_new(Message.INSTRUCT, combined_instruct) self._check_limit() # Add context to the prompt @@ -77,8 +82,9 @@ def make_prompt(self, request: str, for reference_hash in references: prompt = self._store.get_prompt(reference_hash) if not prompt: - logger.error("Reference %s not retrievable while making prompt.", - reference_hash) + logger.error( + "Reference %s not retrievable while making prompt.", reference_hash + ) continue self._prompt.references.append(reference_hash) self._prompt.prepend_history(prompt, self.token_limit) @@ -111,8 +117,10 @@ def iterate_response(self) -> Iterator[str]: try: if hasattr(chunk, "dict"): chunk = chunk.dict() - if "function_call" in chunk["choices"][0]["delta"] and \ - not chunk["choices"][0]["delta"]["function_call"]: + if ( + "function_call" in chunk["choices"][0]["delta"] + and not chunk["choices"][0]["delta"]["function_call"] + ): del chunk["choices"][0]["delta"]["function_call"] if not chunk["choices"][0]["delta"]["content"]: chunk["choices"][0]["delta"]["content"] = "" @@ -123,8 +131,8 @@ def iterate_response(self) -> Iterator[str]: chunk["model"] = config_params["model"] chunk["choices"][0]["index"] = 0 chunk["choices"][0]["finish_reason"] = "stop" - if "role" not in chunk['choices'][0]['delta']: - chunk['choices'][0]['delta']['role']='assistant' + if "role" not in chunk["choices"][0]["delta"]: + chunk["choices"][0]["delta"]["role"] = "assistant" delta = self._prompt.append_response(json.dumps(chunk)) yield delta @@ -136,9 +144,9 @@ def iterate_response(self) -> Iterator[str]: raise RuntimeError("No responses returned from the chat API") if self._need_store: self._store.store_prompt(self._prompt) - yield self._prompt.formatted_footer(0) + '\n' + yield self._prompt.formatted_footer(0) + "\n" for index in range(1, len(self._prompt.responses)): - yield self._prompt.formatted_full_response(index) + '\n' + yield self._prompt.formatted_full_response(index) + "\n" else: response_str = self._chat.complete_response(self._prompt) self._prompt.set_response(response_str) @@ -147,4 +155,4 @@ def iterate_response(self) -> Iterator[str]: if self._need_store: self._store.store_prompt(self._prompt) for index in range(len(self._prompt.responses)): - yield self._prompt.formatted_full_response(index) + '\n' + yield self._prompt.formatted_full_response(index) + "\n" diff --git a/devchat/chat.py b/devchat/chat.py index 9d9003fb..6a3e1d9e 100644 --- a/devchat/chat.py +++ b/devchat/chat.py @@ -1,5 +1,6 @@ from abc import ABC, abstractmethod from typing import Iterator + from devchat.prompt import Prompt diff --git a/devchat/chatmark/chatmark_example/main.py b/devchat/chatmark/chatmark_example/main.py index f834c02c..b7a09ed3 100644 --- a/devchat/chatmark/chatmark_example/main.py +++ b/devchat/chatmark/chatmark_example/main.py @@ -1,7 +1,6 @@ import time - -from devchat.chatmark import Button, Checkbox, Form, Radio, Step, TextEditor # pylint: disable=E402 +from devchat.chatmark import Button, Checkbox, Form, Radio, Step, TextEditor def main(): diff --git a/devchat/chatmark/form.py b/devchat/chatmark/form.py index f610358f..facbf62a 100644 --- a/devchat/chatmark/form.py +++ b/devchat/chatmark/form.py @@ -1,5 +1,3 @@ -# pylint: disable=C0103 -# pylint: disable=W0212 from typing import Dict, List, Optional, Union from .iobase import pipe_interaction diff --git a/devchat/config.py b/devchat/config.py index cdb92ffd..f8bfc627 100644 --- a/devchat/config.py +++ b/devchat/config.py @@ -1,18 +1,21 @@ import os import sys -from typing import List, Dict, Tuple, Optional -from pydantic import BaseModel +from typing import Dict, List, Optional, Tuple + import oyaml as yaml +from pydantic import BaseModel class GeneralProviderConfig(BaseModel): api_key: Optional[str] api_base: Optional[str] + class ModelConfig(BaseModel): max_input_tokens: Optional[int] = sys.maxsize provider: Optional[str] + class GeneralModelConfig(ModelConfig): max_tokens: Optional[int] stop_sequences: Optional[List[str]] @@ -30,7 +33,7 @@ class ChatConfig(BaseModel): class ConfigManager: def __init__(self, dir_path: str): - self.config_path = os.path.join(dir_path, 'config.yml') + self.config_path = os.path.join(dir_path, "config.yml") if not os.path.exists(self.config_path): self._create_sample_file() self._file_is_new = True @@ -47,14 +50,14 @@ def file_last_modified(self) -> float: return os.path.getmtime(self.config_path) def _load_and_validate_config(self) -> ChatConfig: - with open(self.config_path, 'r', encoding='utf-8') as file: + with open(self.config_path, "r", encoding="utf-8") as file: data = yaml.safe_load(file) - if 'providers' in data: - for provider, config in data['providers'].items(): - data['providers'][provider] = GeneralProviderConfig(**config) - for model, config in data['models'].items(): - data['models'][model] = GeneralModelConfig(**config) + if "providers" in data: + for provider, config in data["providers"].items(): + data["providers"][provider] = GeneralProviderConfig(**config) + for model, config in data["models"].items(): + data["models"][model] = GeneralModelConfig(**config) return ChatConfig(**data) @@ -70,9 +73,7 @@ def model_config(self, model_id: Optional[str] = None) -> Tuple[str, ModelConfig return model_id, self.config.models[model_id] def update_model_config( - self, - model_id: str, - new_config: GeneralModelConfig + self, model_id: str, new_config: GeneralModelConfig ) -> GeneralModelConfig: _, old_config = self.model_config(model_id) if new_config.max_input_tokens is not None: @@ -83,46 +84,29 @@ def update_model_config( return self.config.models[model_id] def sync(self): - with open(self.config_path, 'w', encoding='utf-8') as file: + with open(self.config_path, "w", encoding="utf-8") as file: yaml.dump(self.config.dict(exclude_unset=True), file) def _create_sample_file(self): sample_config = ChatConfig( providers={ - "devchat.ai": GeneralProviderConfig( - api_key="" - ), - "openai.com": GeneralProviderConfig( - api_key="" - ), - "general": GeneralProviderConfig( - ) + "devchat.ai": GeneralProviderConfig(api_key=""), + "openai.com": GeneralProviderConfig(api_key=""), + "general": GeneralProviderConfig(), }, models={ "gpt-4": GeneralModelConfig( - max_input_tokens=6000, - provider='devchat.ai', - temperature=0, - stream=True + max_input_tokens=6000, provider="devchat.ai", temperature=0, stream=True ), "gpt-3.5-turbo-16k": GeneralModelConfig( - max_input_tokens=12000, - provider='devchat.ai', - temperature=0, - stream=True + max_input_tokens=12000, provider="devchat.ai", temperature=0, stream=True ), "gpt-3.5-turbo": GeneralModelConfig( - max_input_tokens=3000, - provider='devchat.ai', - temperature=0, - stream=True + max_input_tokens=3000, provider="devchat.ai", temperature=0, stream=True ), - "claude-2": GeneralModelConfig( - provider='general', - max_tokens=20000 - ) + "claude-2": GeneralModelConfig(provider="general", max_tokens=20000), }, - default_model="gpt-3.5-turbo" + default_model="gpt-3.5-turbo", ) - with open(self.config_path, 'w', encoding='utf-8') as file: + with open(self.config_path, "w", encoding="utf-8") as file: yaml.dump(sample_config.dict(exclude_unset=True), file) diff --git a/devchat/engine/__init__.py b/devchat/engine/__init__.py index af78a480..b7102b91 100644 --- a/devchat/engine/__init__.py +++ b/devchat/engine/__init__.py @@ -1,14 +1,14 @@ -from .command_parser import parse_command, Command, CommandParser +from .command_parser import Command, CommandParser, parse_command from .namespace import Namespace from .recursive_prompter import RecursivePrompter -from .router import run_command, load_workflow_instruction +from .router import load_workflow_instruction, run_command __all__ = [ - 'parse_command', - 'Command', - 'CommandParser', - 'Namespace', - 'RecursivePrompter', - 'run_command', - 'load_workflow_instruction' + "parse_command", + "Command", + "CommandParser", + "Namespace", + "RecursivePrompter", + "run_command", + "load_workflow_instruction", ] diff --git a/devchat/engine/command_parser.py b/devchat/engine/command_parser.py index 15ae2181..a024fe1c 100644 --- a/devchat/engine/command_parser.py +++ b/devchat/engine/command_parser.py @@ -1,7 +1,9 @@ import os -from typing import List, Dict, Optional -from pydantic import BaseModel +from typing import Dict, List, Optional + import oyaml as yaml +from pydantic import BaseModel + from .namespace import Namespace @@ -21,7 +23,7 @@ class Command(BaseModel): path: Optional[str] = None -class CommandParser(): +class CommandParser: def __init__(self, namespace: Namespace): self.namespace = namespace @@ -32,7 +34,7 @@ def parse(self, name: str) -> Command: :param name: The command name in the namespace. :return: The JSON representation of the command. """ - file_path = self.namespace.get_file(name, 'command.yml') + file_path = self.namespace.get_file(name, "command.yml") if not file_path: return None return parse_command(file_path) @@ -48,9 +50,9 @@ def parse_command(file_path: str) -> Command: # get path from file_path, /xx1/xx2/xx3.py => /xx1/xx2 config_dir = os.path.dirname(file_path) - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: # replace {curpath} with config_dir - content = file.read().replace('$command_path', config_dir.replace('\\', '/')) + content = file.read().replace("$command_path", config_dir.replace("\\", "/")) config_dict = yaml.safe_load(content) config = Command(**config_dict) config.path = file_path diff --git a/devchat/engine/command_runner.py b/devchat/engine/command_runner.py index 2e893562..e21b0d53 100644 --- a/devchat/engine/command_runner.py +++ b/devchat/engine/command_runner.py @@ -1,37 +1,39 @@ """ Run Command with a input text. """ + +import json import os +import shlex +import subprocess import sys -import json import threading -import subprocess -from typing import List, Dict -import shlex +from typing import Dict, List from devchat.utils import get_logger + from .command_parser import Command from .util import ToolUtil - logger = get_logger(__name__) DEVCHAT_COMMAND_MISS_ERROR_MESSAGE = ( - 'devchat-commands environment is not installed yet. ' - 'Please install it before using the current command.' - 'The devchat-command environment is automatically ' - 'installed after the plugin starts,' - ' and details can be viewed in the output window.' + "devchat-commands environment is not installed yet. " + "Please install it before using the current command." + "The devchat-command environment is automatically " + "installed after the plugin starts," + " and details can be viewed in the output window." ) + def pipe_reader(pipe, out_data, out_flag): while pipe: data = pipe.read(1) - if data == '': + if data == "": break - out_data['out'] += data - print(data, end='', file=out_flag, flush=True) + out_data["out"] += data + print(data, end="", file=out_flag, flush=True) # Equivalent of CommandRun in Python\which executes subprocesses @@ -40,21 +42,25 @@ def __init__(self, model_name: str): self.process = None self._model_name = model_name - def run_command(self, - command_name: str, - command: Command, - history_messages: List[Dict], - input_text: str, - parent_hash: str): + def run_command( + self, + command_name: str, + command: Command, + history_messages: List[Dict], + input_text: str, + parent_hash: str, + ): """ if command has parameters, then generate command parameters from input by LLM if command.input is "required", and input is null, then return error """ - input_text = input_text.strip()\ - .replace(f'/{command_name}', '')\ - .replace('\"', '\\"')\ - .replace('\'', '\\\'')\ - .replace('\n', '\\n') + input_text = ( + input_text.strip() + .replace(f"/{command_name}", "") + .replace('"', '\\"') + .replace("'", "\\'") + .replace("\n", "\\n") + ) arguments = {} if command.parameters and len(command.parameters) > 0: @@ -69,20 +75,19 @@ def run_command(self, return self.run_command_with_parameters( command_name=command_name, command=command, - parameters={ - "input": input_text, - **arguments - }, + parameters={"input": input_text, **arguments}, parent_hash=parent_hash, - history_messages=history_messages + history_messages=history_messages, ) - def run_command_with_parameters(self, - command_name: str, - command: Command, - parameters: Dict[str, str], - parent_hash: str, - history_messages: List[Dict]): + def run_command_with_parameters( + self, + command_name: str, + command: Command, + parameters: Dict[str, str], + parent_hash: str, + history_messages: List[Dict], + ): """ replace $xxx in command.steps[0].run with parameters[xxx] then run command.steps[0].run @@ -91,17 +96,13 @@ def run_command_with_parameters(self, try: env = os.environ.copy() env.update(parameters) - env.update( - self.__load_command_runtime(command) - ) - env.update( - self.__load_chat_data(self._model_name, parent_hash, history_messages) - ) + env.update(self.__load_command_runtime(command)) + env.update(self.__load_chat_data(self._model_name, parent_hash, history_messages)) self.__update_devchat_python_path(env, command.steps[0]["run"]) command_run = command.steps[0]["run"] for parameter in env: - command_run = command_run.replace('$' + parameter, str(env[parameter])) + command_run = command_run.replace("$" + parameter, str(env[parameter])) if self.__check_command_python_error(command_run, env): return result @@ -124,14 +125,15 @@ def __run_command_with_thread_output(self, command_str: str, env: Dict[str, str] """ run command string """ + def handle_output(process): - stdout_data, stderr_data = {'out': ''}, {'out': ''} + stdout_data, stderr_data = {"out": ""}, {"out": ""} stdout_thread = threading.Thread( - target=pipe_reader, - args=(process.stdout, stdout_data, sys.stdout)) + target=pipe_reader, args=(process.stdout, stdout_data, sys.stdout) + ) stderr_thread = threading.Thread( - target=pipe_reader, - args=(process.stderr, stderr_data, sys.stderr)) + target=pipe_reader, args=(process.stderr, stderr_data, sys.stderr) + ) stdout_thread.start() stderr_thread.start() stdout_thread.join() @@ -142,17 +144,17 @@ def handle_output(process): if isinstance(env[key], (List, Dict)): env[key] = json.dumps(env[key]) with subprocess.Popen( - shlex.split(command_str), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - text=True - ) as process: + shlex.split(command_str), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True, + ) as process: return handle_output(process) def __check_command_python_error(self, command_run: str, parameters: Dict[str, str]): - need_command_python = command_run.find('$command_python ') != -1 - has_command_python = parameters.get('command_python', None) + need_command_python = command_run.find("$command_python ") != -1 + has_command_python = parameters.get("command_python", None) if need_command_python and not has_command_python: print(DEVCHAT_COMMAND_MISS_ERROR_MESSAGE, file=sys.stderr, flush=True) @@ -162,9 +164,9 @@ def __check_command_python_error(self, command_run: str, parameters: Dict[str, s def __get_readme(self, command: Command): try: command_dir = os.path.dirname(command.path) - readme_file = os.path.join(command_dir, 'README.md') + readme_file = os.path.join(command_dir, "README.md") if os.path.exists(readme_file): - with open(readme_file, 'r', encoding='utf8') as file: + with open(readme_file, "r", encoding="utf8") as file: readme = file.read() return readme return None @@ -172,8 +174,8 @@ def __get_readme(self, command: Command): return None def __check_input_miss_error( - self, command: Command, command_name: str, parameters: Dict[str, str] - ): + self, command: Command, command_name: str, parameters: Dict[str, str] + ): is_input_required = command.input == "required" if not (is_input_required and parameters["input"] == ""): return False @@ -197,7 +199,7 @@ def __check_parameters_miss_error(self, command: Command, command_run: str): missed_parameters = [] for parameter_name in parameter_names: - if command_run.find('$' + parameter_name) != -1: + if command_run.find("$" + parameter_name) != -1: missed_parameters.append(parameter_name) if len(missed_parameters) == 0: @@ -216,22 +218,21 @@ def __load_command_runtime(self, command: Command): # visit each path in command_path, for example: /usr/x1/x2/x3 # then load visit: /usr, /usr/x1, /usr/x1/x2, /usr/x1/x2/x3 - paths = command_path.split('/') - for index in range(1, len(paths)+1): + paths = command_path.split("/") + for index in range(1, len(paths) + 1): try: - path = '/'.join(paths[:index]) - runtime_file = os.path.join(path, 'runtime.json') + path = "/".join(paths[:index]) + runtime_file = os.path.join(path, "runtime.json") if os.path.exists(runtime_file): - with open(runtime_file, 'r', encoding='utf8') as file: + with open(runtime_file, "r", encoding="utf8") as file: command_runtime_config = json.loads(file.read()) runtime_config.update(command_runtime_config) except Exception: pass # for windows - if runtime_config.get('command_python', None): - runtime_config['command_python'] = \ - runtime_config['command_python'].replace('\\', '/') + if runtime_config.get("command_python", None): + runtime_config["command_python"] = runtime_config["command_python"].replace("\\", "/") return runtime_config def __load_chat_data(self, model_name: str, parent_hash: str, history_messages: List[Dict]): @@ -242,16 +243,15 @@ def __load_chat_data(self, model_name: str, parent_hash: str, history_messages: } def __update_devchat_python_path(self, env: Dict[str, str], command_run: str): - python_path = os.environ.get('PYTHONPATH', '') - env['DEVCHAT_PYTHONPATH'] = os.environ.get('DEVCHAT_PYTHONPATH', python_path) - if command_run.find('$devchat_python ') == -1: - del env['PYTHONPATH'] - env["devchat_python"] = sys.executable.replace('\\', '/') - - def _call_function_by_llm(self, - command_name: str, - command: Command, - history_messages: List[Dict]): + python_path = os.environ.get("PYTHONPATH", "") + env["DEVCHAT_PYTHONPATH"] = os.environ.get("DEVCHAT_PYTHONPATH", python_path) + if command_run.find("$devchat_python ") == -1: + del env["PYTHONPATH"] + env["devchat_python"] = sys.executable.replace("\\", "/") + + def _call_function_by_llm( + self, command_name: str, command: Command, history_messages: List[Dict] + ): """ command needs multi parameters, so we need parse each parameter by LLM from input_text diff --git a/devchat/engine/namespace.py b/devchat/engine/namespace.py index ba6fc71a..cce1a228 100644 --- a/devchat/engine/namespace.py +++ b/devchat/engine/namespace.py @@ -1,17 +1,16 @@ import os -from typing import List, Optional import re +from typing import List, Optional class Namespace: - def __init__(self, root_path: str, - branches: List[str] = None): + def __init__(self, root_path: str, branches: List[str] = None): """ :param root_path: The root path of the namespace. :param branches: The hidden branches with ascending order of priority. """ self.root_path = root_path - self.branches = branches if branches else ['sys', 'org', 'usr'] + self.branches = branches if branches else ["sys", "org", "usr"] @staticmethod def is_valid_name(name: str) -> bool: @@ -28,7 +27,7 @@ def is_valid_name(name: str) -> bool: # The regular expression pattern for a valid name if name is None: return False - pattern = r'^$|^(?!.*\.\.)[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$' + pattern = r"^$|^(?!.*\.\.)[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$" return bool(re.match(pattern, name)) def get_file(self, name: str, file: str) -> Optional[str]: @@ -40,7 +39,7 @@ def get_file(self, name: str, file: str) -> Optional[str]: if not self.is_valid_name(name): return None # Convert the dot-separated name to a path - path = os.path.join(*name.split('.')) + path = os.path.join(*name.split(".")) for branch in reversed(self.branches): full_path = os.path.join(self.root_path, branch, path) if os.path.isdir(full_path): @@ -60,7 +59,7 @@ def list_files(self, name: str) -> List[str]: if not self.is_valid_name(name): raise ValueError(f"Invalid name to list files: {name}") # Convert the dot-separated name to a path - path = os.path.join(*name.split('.')) + path = os.path.join(*name.split(".")) files = {} path_found = False for branch in self.branches: @@ -77,7 +76,7 @@ def list_files(self, name: str) -> List[str]: # Sort the files in alphabetical order before returning return sorted(files.values()) if files else [] - def list_names(self, name: str = '', recursive: bool = False) -> List[str]: + def list_names(self, name: str = "", recursive: bool = False) -> List[str]: """ :param name: The command name in the namespace. Defaults to the root. :param recursive: Whether to list all descendant names or only child names. @@ -86,7 +85,7 @@ def list_names(self, name: str = '', recursive: bool = False) -> List[str]: if not self.is_valid_name(name): raise ValueError(f"Invalid name to list names: {name}") commands = set() - path = os.path.join(*name.split('.')) + path = os.path.join(*name.split(".")) found = False for branch in self.branches: full_path = os.path.join(self.root_path, branch, path) @@ -101,10 +100,10 @@ def list_names(self, name: str = '', recursive: bool = False) -> List[str]: def _add_dirnames_to_commands(self, full_path: str, name: str, commands: set): for dirname in os.listdir(full_path): - if dirname.startswith('.'): + if dirname.startswith("."): continue if os.path.isdir(os.path.join(full_path, dirname)): - command_name = '.'.join([name, dirname]) if name else dirname + command_name = ".".join([name, dirname]) if name else dirname commands.add(command_name) def _add_recursive_dirnames_to_commands(self, full_path: str, name: str, commands: set): @@ -112,10 +111,10 @@ def _add_recursive_dirnames_to_commands(self, full_path: str, name: str, command def _recursive_dir_walk(self, full_path: str, name: str, commands: set): for dirname in os.listdir(full_path): - if dirname.startswith('.'): + if dirname.startswith("."): continue dir_path = os.path.join(full_path, dirname) if os.path.isdir(dir_path): - command_name = '.'.join([name, dirname]) if name else dirname + command_name = ".".join([name, dirname]) if name else dirname commands.add(command_name) self._recursive_dir_walk(dir_path, command_name, commands) diff --git a/devchat/engine/recursive_prompter.py b/devchat/engine/recursive_prompter.py index e5150a09..cdff8f17 100644 --- a/devchat/engine/recursive_prompter.py +++ b/devchat/engine/recursive_prompter.py @@ -1,5 +1,6 @@ -import re import os +import re + from .namespace import Namespace @@ -8,30 +9,30 @@ def __init__(self, namespace: Namespace): self.namespace = namespace def run(self, name: str) -> str: - ancestors = name.split('.') - merged_content = '' + ancestors = name.split(".") + merged_content = "" for index in range(len(ancestors)): - ancestor_name = '.'.join(ancestors[:index + 1]) - file_path = self.namespace.get_file(ancestor_name, 'prompt.txt') + ancestor_name = ".".join(ancestors[: index + 1]) + file_path = self.namespace.get_file(ancestor_name, "prompt.txt") if file_path: - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: prompt_content = file.read() # replace @file@ with the content of the file prompt_content = self._replace_file_references(file_path, prompt_content) merged_content += prompt_content - merged_content += '\n' + merged_content += "\n" return merged_content def _replace_file_references(self, prompt_file_path: str, content: str) -> str: # prompt_file_path is the path to the file that contains the content # @relative file path@: file is relative to the prompt_file_path - pattern = re.compile(r'@(.+?)@') + pattern = re.compile(r"@(.+?)@") matches = pattern.findall(content) for match in matches: file_path = os.path.join(os.path.dirname(prompt_file_path), match) if os.path.exists(file_path): - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: file_content = file.read() - content = content.replace(f'@{match}@', file_content) + content = content.replace(f"@{match}@", file_content) return content diff --git a/devchat/engine/router.py b/devchat/engine/router.py index 786f7c49..da6094b1 100644 --- a/devchat/engine/router.py +++ b/devchat/engine/router.py @@ -1,18 +1,20 @@ import os from typing import List + from .command_runner import CommandRunner -from .util import CommandUtil from .namespace import Namespace -from.recursive_prompter import RecursivePrompter +from .recursive_prompter import RecursivePrompter +from .util import CommandUtil + def load_workflow_instruction(user_input: str): user_input = user_input.strip() if len(user_input) == 0: return None - if user_input[:1] != '/': + if user_input[:1] != "/": return None - workflows_dir = os.path.join(os.path.expanduser('~/.chat'), 'workflows') + workflows_dir = os.path.join(os.path.expanduser("~/.chat"), "workflows") if not os.path.exists(workflows_dir): return None if not os.path.isdir(workflows_dir): @@ -28,19 +30,16 @@ def load_workflow_instruction(user_input: str): def run_command( - model_name: str, - history_messages: List[dict], - input_text: str, - parent_hash: str, - auto_fun: bool): + model_name: str, history_messages: List[dict], input_text: str, parent_hash: str, auto_fun: bool +): """ load command config, and then run Command """ # split input_text by ' ','\n','\t' if len(input_text.strip()) == 0: return None - if input_text.strip()[:1] != '/': - if not (auto_fun and model_name.startswith('gpt-')): + if input_text.strip()[:1] != "/": + if not (auto_fun and model_name.startswith("gpt-")): return None # TODO @@ -60,5 +59,5 @@ def run_command( command=command_obj, history_messages=history_messages, input_text=input_text, - parent_hash=parent_hash + parent_hash=parent_hash, ) diff --git a/devchat/engine/util.py b/devchat/engine/util.py index ded2d9c8..93d9d02e 100644 --- a/devchat/engine/util.py +++ b/devchat/engine/util.py @@ -1,26 +1,25 @@ -# pylint: disable=import-outside-toplevel +import json import os import sys -import json -from typing import List, Dict +from typing import Dict, List from devchat._cli.utils import init_dir from devchat.utils import get_logger +from .command_parser import Command, CommandParser from .namespace import Namespace -from .command_parser import CommandParser, Command - logger = get_logger(__name__) DEFAULT_MODEL = "gpt-3.5-turbo" + class CommandUtil: @staticmethod def __command_parser(): _, user_chat_dir = init_dir() - workflows_dir = os.path.join(user_chat_dir, 'workflows') + workflows_dir = os.path.join(user_chat_dir, "workflows") if not os.path.exists(workflows_dir) or not os.path.isdir(workflows_dir): return None @@ -42,8 +41,8 @@ def load_commands() -> List[Command]: return [] command_names = commander.namespace.list_names("", True) - commands = [ (name, commander.parse(name)) for name in command_names ] - return [ cmd for cmd in commands if cmd[1] ] + commands = [(name, commander.parse(name)) for name in command_names] + return [cmd for cmd in commands if cmd[1]] class ToolUtil: @@ -56,23 +55,20 @@ def __make_function_parameters(command: Command): for key, value in command.parameters.items(): properties[key] = {} for key1, value1 in value.dict().items(): - if key1 not in ['type', 'description', 'enum'] or value1 is None: + if key1 not in ["type", "description", "enum"] or value1 is None: continue properties[key][key1] = value1 required.append(key) - elif command.steps[0]['run'].find('$input') > 0: - properties['input'] = { - "type": "string", - "description": "input text" - } - required.append('input') + elif command.steps[0]["run"].find("$input") > 0: + properties["input"] = {"type": "string", "description": "input text"} + required.append("input") return properties, required @staticmethod def make_function(command: Command, command_name: str): properties, required = ToolUtil.__make_function_parameters(command) - command_name = command_name.replace('.', '---') + command_name = command_name.replace(".", "---") return { "type": "function", @@ -84,37 +80,37 @@ def make_function(command: Command, command_name: str): "properties": properties, "required": required, }, - } + }, } @staticmethod def select_function_by_llm( - history_messages: List[Dict], tools: List[Dict], model: str = DEFAULT_MODEL - ): - import openai + history_messages: List[Dict], tools: List[Dict], model: str = DEFAULT_MODEL + ): import httpx + import openai + proxy_url = os.environ.get("DEVCHAT_PROXY", "") - proxy_setting ={"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + proxy_setting = ( + {"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + ) client = openai.OpenAI( api_key=os.environ.get("OPENAI_API_KEY", None), base_url=os.environ.get("OPENAI_API_BASE", None), - http_client=httpx.Client(**proxy_setting, trust_env=False) + http_client=httpx.Client(**proxy_setting, trust_env=False), ) try: response = client.chat.completions.create( - messages=history_messages, - model=model, - stream=False, - tools=tools + messages=history_messages, model=model, stream=False, tools=tools ) respose_message = response.dict()["choices"][0]["message"] - if not respose_message['tool_calls']: + if not respose_message["tool_calls"]: return None - tool_call = respose_message['tool_calls'][0]['function'] - if tool_call['name'] != tools[0]["function"]["name"]: + tool_call = respose_message["tool_calls"][0]["function"] + if tool_call["name"] != tools[0]["function"]["name"]: error_msg = ( "The LLM returned an invalid function name. " f"Expected: {tools[0]['function']['name']}, " @@ -123,8 +119,8 @@ def select_function_by_llm( print(error_msg, file=sys.stderr, flush=True) return None return { - "name": tool_call['name'].replace('---', '.'), - "arguments": json.loads(tool_call['arguments']) + "name": tool_call["name"].replace("---", "."), + "arguments": json.loads(tool_call["arguments"]), } except (ConnectionError, openai.APIConnectionError) as err: print("ConnectionError:", err, file=sys.stderr, flush=True) @@ -139,25 +135,22 @@ def select_function_by_llm( return None @staticmethod - def _create_tool(command_name:str, command: Command) -> dict: + def _create_tool(command_name: str, command: Command) -> dict: properties = {} required = [] if command.parameters: for key, value in command.parameters.items(): properties[key] = {} for key1, value1 in value.dict().items(): - if key1 not in ['type', 'description', 'enum'] or value1 is None: + if key1 not in ["type", "description", "enum"] or value1 is None: continue properties[key][key1] = value1 required.append(key) - elif command.steps[0]['run'].find('$input') > 0: - properties['input'] = { - "type": "string", - "description": "input text" - } - required.append('input') + elif command.steps[0]["run"].find("$input") > 0: + properties["input"] = {"type": "string", "description": "input text"} + required.append("input") - command_name = command_name.replace('.', '---') + command_name = command_name.replace(".", "---") return { "type": "function", "function": { @@ -168,6 +161,5 @@ def _create_tool(command_name:str, command: Command) -> dict: "properties": properties, "required": required, }, - } + }, } - \ No newline at end of file diff --git a/devchat/ide/__init__.py b/devchat/ide/__init__.py index 986fe28c..95791c80 100644 --- a/devchat/ide/__init__.py +++ b/devchat/ide/__init__.py @@ -1,5 +1,5 @@ from .service import IDEService -from .types import * +from .types import * # noqa: F403 from .types import __all__ as types_all __all__ = types_all + [ diff --git a/devchat/ide/idea_services.py b/devchat/ide/idea_services.py index e0211316..db5753ae 100644 --- a/devchat/ide/idea_services.py +++ b/devchat/ide/idea_services.py @@ -1,5 +1,6 @@ -from .types import LocationWithText from .rpc import rpc_method +from .types import LocationWithText + class IdeaIDEService: def __init__(self): @@ -7,8 +8,8 @@ def __init__(self): @rpc_method def get_visible_range(self) -> LocationWithText: - return LocationWithText.parse_obj(self._result) + return LocationWithText.parse_obj(self._result) @rpc_method def get_selected_range(self) -> LocationWithText: - return LocationWithText.parse_obj(self._result) + return LocationWithText.parse_obj(self._result) diff --git a/devchat/ide/rpc.py b/devchat/ide/rpc.py index b8f56cc0..67e2923e 100644 --- a/devchat/ide/rpc.py +++ b/devchat/ide/rpc.py @@ -1,8 +1,3 @@ -# pylint: disable=C0103 -# pylint: disable=W3101 -# pylint: disable=W0719 -# pylint: disable=R1710 -# pylint: disable=W0212 import os from functools import wraps diff --git a/devchat/ide/service.py b/devchat/ide/service.py index ce477e1d..1f9929ef 100644 --- a/devchat/ide/service.py +++ b/devchat/ide/service.py @@ -1,17 +1,9 @@ -# disable pylint -# pylint: disable=W0613 -# pylint: disable=E1133 -# pylint: disable=R1710 -# pylint: disable=W0719 -# pylint: disable=W3101 -# pylint: disable=C0103 - from typing import List +from .idea_services import IdeaIDEService from .rpc import rpc_method -from .types import Location, SymbolNode, LocationWithText +from .types import Location, LocationWithText, SymbolNode from .vscode_services import selected_range, visible_range -from .idea_services import IdeaIDEService class IDEService: @@ -39,7 +31,7 @@ def get_lsp_brige_port(self) -> str: @rpc_method def install_python_env(self, command_name: str, requirements_file: str) -> str: """ - A method to install a Python environment with the provided command name + A method to install a Python environment with the provided command name and requirements file, returning python path installed. Command name is the name of the environment to be installed. """ @@ -136,7 +128,7 @@ def get_visible_range(self) -> LocationWithText: Determines and returns the visible range of code in the current IDE. Returns: - A tuple denoting the visible range if the IDE is VSCode, or defers to + A tuple denoting the visible range if the IDE is VSCode, or defers to IdeaIDEService's get_visible_range method for other IDEs. """ if self.ide_name() == "vscode": diff --git a/devchat/ide/types.py b/devchat/ide/types.py index c999e53b..853048b2 100644 --- a/devchat/ide/types.py +++ b/devchat/ide/types.py @@ -2,13 +2,7 @@ from pydantic import BaseModel -__all__ = [ - "Position", - "Range", - "Location", - "SymbolNode", - "LocationWithText" -] +__all__ = ["Position", "Range", "Location", "SymbolNode", "LocationWithText"] class Position(BaseModel): diff --git a/devchat/ide/vscode_services.py b/devchat/ide/vscode_services.py index 838a7a22..c8f93a82 100644 --- a/devchat/ide/vscode_services.py +++ b/devchat/ide/vscode_services.py @@ -5,12 +5,12 @@ @rpc_call -def run_code(code: str): # pylint: disable=unused-argument +def run_code(code: str): pass @rpc_call -def diff_apply(filepath, content): # pylint: disable=unused-argument +def diff_apply(filepath, content): pass @@ -110,6 +110,7 @@ def visible_lines(): "visibleRange": [start_line, end_line], } + def visible_range() -> LocationWithText: visible_range_text = visible_lines() return LocationWithText( @@ -124,7 +125,7 @@ def visible_range() -> LocationWithText: "line": visible_range_text["visibleRange"][1], "character": 0, }, - } + }, ) @@ -159,6 +160,7 @@ def selected_lines(): "selectedRange": [start_line, start_col, end_line, end_col], } + def selected_range() -> LocationWithText: selected_range_text = selected_lines() return LocationWithText( @@ -173,5 +175,5 @@ def selected_range() -> LocationWithText: "line": selected_range_text["selectedRange"][2], "character": selected_range_text["selectedRange"][3], }, - } + }, ) diff --git a/devchat/llm/chat.py b/devchat/llm/chat.py index 8252e901..68121b3d 100644 --- a/devchat/llm/chat.py +++ b/devchat/llm/chat.py @@ -47,7 +47,7 @@ def chat( ): def decorator(func): @wraps(func) - def wrapper(*args, **kwargs): # pylint: disable=unused-argument + def wrapper(*args, **kwargs): nonlocal prompt, memory, model, llm_config prompt_new = prompt.format(**kwargs) messages = memory.contexts() if memory else [] @@ -86,7 +86,7 @@ def chat_json( ): def decorator(func): @wraps(func) - def wrapper(*args, **kwargs): # pylint: disable=unused-argument + def wrapper(*args, **kwargs): nonlocal prompt, memory, model, llm_config prompt_new = prompt.format(**kwargs) messages = memory.contexts() if memory else [] diff --git a/devchat/llm/openai.py b/devchat/llm/openai.py index 4760ac50..f6dfd03f 100644 --- a/devchat/llm/openai.py +++ b/devchat/llm/openai.py @@ -41,12 +41,12 @@ def chat_completion_stream_commit( llm_config: Dict, # {"model": "...", ...} ): proxy_url = os.environ.get("DEVCHAT_PROXY", "") - proxy_setting ={"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + proxy_setting = {"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} client = openai.OpenAI( api_key=os.environ.get("OPENAI_API_KEY", None), base_url=os.environ.get("OPENAI_API_BASE", None), - http_client=httpx.Client(**proxy_setting, trust_env=False) + http_client=httpx.Client(**proxy_setting, trust_env=False), ) llm_config["stream"] = True @@ -56,12 +56,12 @@ def chat_completion_stream_commit( def chat_completion_stream_raw(**kwargs): proxy_url = os.environ.get("DEVCHAT_PROXY", "") - proxy_setting ={"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + proxy_setting = {"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} client = openai.OpenAI( api_key=os.environ.get("OPENAI_API_KEY", None), base_url=os.environ.get("OPENAI_API_BASE", None), - http_client=httpx.Client(**proxy_setting, trust_env=False) + http_client=httpx.Client(**proxy_setting, trust_env=False), ) kwargs["stream"] = True @@ -87,7 +87,7 @@ def retry_timeout(chunks): def chunk_list(chunks): - return [chunk for chunk in chunks] # pylint: disable=R1721 + return [chunk for chunk in chunks] def chunks_content(chunks): @@ -164,13 +164,13 @@ def to_dict_content_and_call(content, tool_calls=None): exception_output_handle(lambda err: None), ) -def chat_completion_no_stream_return_json( - messages: List[Dict], llm_config: Dict): + +def chat_completion_no_stream_return_json(messages: List[Dict], llm_config: Dict): """call llm without stream, return json object""" - llm_config["response_format"]={"type": "json_object"} + llm_config["response_format"] = {"type": "json_object"} return chat_completion_no_stream_return_json_with_retry( - messages=messages, - llm_config=llm_config) + messages=messages, llm_config=llm_config + ) chat_completion_stream = exception_handle( diff --git a/devchat/llm/pipeline.py b/devchat/llm/pipeline.py index 2a4c867f..7917f0b4 100644 --- a/devchat/llm/pipeline.py +++ b/devchat/llm/pipeline.py @@ -19,8 +19,7 @@ def wrapper(*args, **kwargs): raise err.error continue except Exception as err: - raise err - raise err.error + raise err.error return wrapper @@ -62,9 +61,8 @@ def pipeline(*funcs): def wrapper(*args, **kwargs): for index, func in enumerate(funcs): if index > 0: - # pylint: disable=E1101 if isinstance(args, Dict) and args.get("__type__", None) == "parallel": - args = func(*args["value"]) # pylint: disable=E1126 + args = func(*args["value"]) else: args = func(args) else: diff --git a/devchat/llm/tools_call.py b/devchat/llm/tools_call.py index 01a189e1..9ab397df 100644 --- a/devchat/llm/tools_call.py +++ b/devchat/llm/tools_call.py @@ -3,9 +3,9 @@ import sys from functools import wraps -from devchat.memory import ChatMemory from devchat.chatmark import Form, Radio, TextEditor from devchat.ide import IDEService +from devchat.memory import ChatMemory from .openai import chat_call_completion_stream @@ -140,7 +140,7 @@ def chat_tools( ): def decorator(func): @wraps(func) - def wrapper(*args, **kwargs): # pylint: disable=unused-argument + def wrapper(*args, **kwargs): nonlocal prompt, memory, model, tools, call_confirm_fun, llm_config prompt = prompt.format(**kwargs) if not tools: diff --git a/devchat/memory/__init__.py b/devchat/memory/__init__.py index 4a5905cc..8496fd54 100644 --- a/devchat/memory/__init__.py +++ b/devchat/memory/__init__.py @@ -1,4 +1,3 @@ - from .base import ChatMemory from .fixsize_memory import FixSizeChatMemory diff --git a/devchat/message.py b/devchat/message.py index 25aa9b44..e485a000 100644 --- a/devchat/message.py +++ b/devchat/message.py @@ -7,6 +7,7 @@ class Message(ABC): """ The basic unit of information in a prompt. """ + content: str = "" INSTRUCT = "instruct" @@ -22,7 +23,7 @@ def to_dict(self) -> dict: @classmethod @abstractmethod - def from_dict(cls, message_data: dict) -> 'Message': + def from_dict(cls, message_data: dict) -> "Message": """ Convert the message from a dictionary. """ diff --git a/devchat/openai/__init__.py b/devchat/openai/__init__.py index cc0ad3c4..23a7569a 100644 --- a/devchat/openai/__init__.py +++ b/devchat/openai/__init__.py @@ -1,11 +1,11 @@ -from .openai_chat import OpenAIChatParameters, OpenAIChatConfig, OpenAIChat +from .openai_chat import OpenAIChat, OpenAIChatConfig, OpenAIChatParameters from .openai_message import OpenAIMessage from .openai_prompt import OpenAIPrompt __all__ = [ - 'OpenAIChat', - 'OpenAIChatConfig', - 'OpenAIChatParameters', - 'OpenAIMessage', - 'OpenAIPrompt' + "OpenAIChat", + "OpenAIChatConfig", + "OpenAIChatParameters", + "OpenAIMessage", + "OpenAIPrompt", ] diff --git a/devchat/openai/http_openai.py b/devchat/openai/http_openai.py index c405b315..5c1c7a24 100755 --- a/devchat/openai/http_openai.py +++ b/devchat/openai/http_openai.py @@ -4,6 +4,7 @@ import sys from urllib.parse import urlparse + class LineReader: def __init__(self, response): self.response = response @@ -18,7 +19,7 @@ def __next__(self): line = line.strip() if not line: return self.__next__() - line = line.decode('utf-8') + line = line.decode("utf-8") if not line.startswith("data:"): print("Receive invalid line: {line}", end="\n\n", file=sys.stderr) raise ValueError(f"Invalid line: {line}") @@ -31,13 +32,17 @@ def __next__(self): print(f"Error decoding JSON: {err}", end="\n\n", file=sys.stderr) raise ValueError(f"Invalid line: {line}") from err + def stream_response(connection: http.client.HTTPSConnection, data, headers): connection.request("POST", "/v1/chat/completions", body=json.dumps(data), headers=headers) response = connection.getresponse() if response.status != 200: - print(f"Error: {response.status} - {response.reason} {response.read()}", - end="\n\n", file=sys.stderr) + print( + f"Error: {response.status} - {response.reason} {response.read()}", + end="\n\n", + file=sys.stderr, + ) return None return LineReader(response=response) diff --git a/devchat/openai/openai_chat.py b/devchat/openai/openai_chat.py index 6426f904..46cf2456 100755 --- a/devchat/openai/openai_chat.py +++ b/devchat/openai/openai_chat.py @@ -1,16 +1,18 @@ -# pylint: disable=import-outside-toplevel import json import os -from typing import Optional, Union, List, Dict, Iterator +from typing import Dict, Iterator, List, Optional, Union + from pydantic import BaseModel, Field from devchat.chat import Chat from devchat.utils import get_user_info, user_id + +from .http_openai import stream_request from .openai_message import OpenAIMessage from .openai_prompt import OpenAIPrompt -from .http_openai import stream_request -class OpenAIChatParameters(BaseModel, extra='ignore'): + +class OpenAIChatParameters(BaseModel, extra="ignore"): temperature: Optional[float] = Field(0, ge=0, le=2) top_p: Optional[float] = Field(None, ge=0, le=1) n: Optional[int] = Field(None, ge=1) @@ -28,6 +30,7 @@ class OpenAIChatConfig(OpenAIChatParameters): """ Configuration object for the OpenAIChat APIs. """ + model: str @@ -35,6 +38,7 @@ class OpenAIChat(Chat): """ OpenAIChat class that handles communication with the OpenAI Chat API. """ + def __init__(self, config: OpenAIChatConfig): """ Initialize the OpenAIChat class with a configuration object. @@ -52,83 +56,81 @@ def init_prompt(self, request: str, function_name: Optional[str] = None) -> Open return prompt def load_prompt(self, data: dict) -> OpenAIPrompt: - data['_new_messages'] = { + data["_new_messages"] = { k: [OpenAIMessage.from_dict(m) for m in v] - if isinstance(v, list) else OpenAIMessage.from_dict(v) - for k, v in data['_new_messages'].items() if k != 'function' + if isinstance(v, list) + else OpenAIMessage.from_dict(v) + for k, v in data["_new_messages"].items() + if k != "function" + } + data["_history_messages"] = { + k: [OpenAIMessage.from_dict(m) for m in v] for k, v in data["_history_messages"].items() } - data['_history_messages'] = {k: [OpenAIMessage.from_dict(m) for m in v] - for k, v in data['_history_messages'].items()} return OpenAIPrompt(**data) def complete_response(self, prompt: OpenAIPrompt) -> str: - import openai import httpx + import openai # Filter the config parameters with set values config_params = self.config.dict(exclude_unset=True) if prompt.get_functions(): - config_params['functions'] = prompt.get_functions() - config_params['function_call'] = 'auto' - config_params['stream'] = False + config_params["functions"] = prompt.get_functions() + config_params["function_call"] = "auto" + config_params["stream"] = False proxy_url = os.environ.get("DEVCHAT_PROXY", "") - proxy_setting ={"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + proxy_setting = ( + {"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + ) client = openai.OpenAI( api_key=os.environ.get("OPENAI_API_KEY", None), base_url=os.environ.get("OPENAI_API_BASE", None), - http_client=httpx.Client(**proxy_setting, trust_env=False) + http_client=httpx.Client(**proxy_setting, trust_env=False), ) - response = client.chat.completions.create( - messages=prompt.messages, - **config_params - ) + response = client.chat.completions.create(messages=prompt.messages, **config_params) if isinstance(response, openai.types.chat.chat_completion.ChatCompletion): return json.dumps(response.dict()) return str(response) def stream_response(self, prompt: OpenAIPrompt) -> Iterator: - api_key=os.environ.get("OPENAI_API_KEY", None) - base_url=os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1/") + api_key = os.environ.get("OPENAI_API_KEY", None) + base_url = os.environ.get("OPENAI_API_BASE", "https://api.openai.com/v1/") if not os.environ.get("USE_TIKTOKEN", False) and base_url != "https://api.openai.com/v1/": config_params = self.config.dict(exclude_unset=True) if prompt.get_functions(): - config_params['functions'] = prompt.get_functions() - config_params['function_call'] = 'auto' - config_params['stream'] = True - - data = { - "messages":prompt.messages, - **config_params, - "timeout":180 - } + config_params["functions"] = prompt.get_functions() + config_params["function_call"] = "auto" + config_params["stream"] = True + + data = {"messages": prompt.messages, **config_params, "timeout": 180} response = stream_request(api_key, base_url, data) return response - import openai import httpx + import openai # Filter the config parameters with set values config_params = self.config.dict(exclude_unset=True) if prompt.get_functions(): - config_params['functions'] = prompt.get_functions() - config_params['function_call'] = 'auto' - config_params['stream'] = True + config_params["functions"] = prompt.get_functions() + config_params["function_call"] = "auto" + config_params["stream"] = True proxy_url = os.environ.get("DEVCHAT_PROXY", "") - proxy_setting ={"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + proxy_setting = ( + {"proxy": {"https://": proxy_url, "http://": proxy_url}} if proxy_url else {} + ) client = openai.OpenAI( api_key=os.environ.get("OPENAI_API_KEY", None), base_url=os.environ.get("OPENAI_API_BASE", None), - http_client=httpx.Client(**proxy_setting, trust_env=False) + http_client=httpx.Client(**proxy_setting, trust_env=False), ) response = client.chat.completions.create( - messages=prompt.messages, - **config_params, - timeout=180 + messages=prompt.messages, **config_params, timeout=180 ) return response diff --git a/devchat/openai/openai_message.py b/devchat/openai/openai_message.py index f36c373b..6a9afa1a 100644 --- a/devchat/openai/openai_message.py +++ b/devchat/openai/openai_message.py @@ -1,6 +1,6 @@ import ast import json -from dataclasses import dataclass, asdict, field, fields +from dataclasses import asdict, dataclass, field, fields from typing import Dict, Optional from devchat.message import Message @@ -17,19 +17,21 @@ def __post_init__(self): raise ValueError("Invalid role. Must be one of 'system', 'user', or 'assistant'.") if not self._validate_name(): - raise ValueError("Invalid name. Must contain a-z, A-Z, 0-9, and underscores, " - "with a maximum length of 64 characters.") + raise ValueError( + "Invalid name. Must contain a-z, A-Z, 0-9, and underscores, " + "with a maximum length of 64 characters." + ) def to_dict(self) -> dict: state = asdict(self) - if state['name'] is None: - del state['name'] - if not state['function_call'] or len(state['function_call'].keys()) == 0: - del state['function_call'] + if state["name"] is None: + del state["name"] + if not state["function_call"] or len(state["function_call"].keys()) == 0: + del state["function_call"] return state @classmethod - def from_dict(cls, message_data: dict) -> 'OpenAIMessage': + def from_dict(cls, message_data: dict) -> "OpenAIMessage": keys = {f.name for f in fields(cls)} kwargs = {k: v for k, v in message_data.items() if k in keys} return cls(**kwargs) @@ -44,24 +46,24 @@ def function_call_to_json(self): } ''' if not self.function_call: - return '' + return "" function_call_copy = self.function_call.copy() - if 'arguments' in function_call_copy: + if "arguments" in function_call_copy: # arguments field may be not a json string # we can try parse it by eval try: - function_call_copy['arguments'] = ast.literal_eval(function_call_copy['arguments']) + function_call_copy["arguments"] = ast.literal_eval(function_call_copy["arguments"]) except Exception: # if it is not a json string, we can do nothing try: - function_call_copy['arguments'] = json.loads(function_call_copy['arguments']) + function_call_copy["arguments"] = json.loads(function_call_copy["arguments"]) except Exception: pass - return '```command\n' + json.dumps(function_call_copy) + '\n```' + return "```command\n" + json.dumps(function_call_copy) + "\n```" def stream_from_dict(self, message_data: dict) -> str: """Append to the message from a dictionary returned from a streaming chat API.""" - delta = message_data.get('content', '') + delta = message_data.get("content", "") if self.content: self.content += delta else: diff --git a/devchat/openai/openai_prompt.py b/devchat/openai/openai_prompt.py index 0034b548..fe71bdd1 100644 --- a/devchat/openai/openai_prompt.py +++ b/devchat/openai/openai_prompt.py @@ -1,11 +1,12 @@ -from dataclasses import dataclass import json import sys +from dataclasses import dataclass from typing import List, Optional -from devchat.prompt import Prompt + from devchat.message import Message -from devchat.utils import update_dict, get_logger -from devchat.utils import openai_message_tokens, openai_response_tokens +from devchat.prompt import Prompt +from devchat.utils import get_logger, openai_message_tokens, openai_response_tokens, update_dict + from .openai_message import OpenAIMessage logger = get_logger(__name__) @@ -31,9 +32,10 @@ def messages(self) -> List[dict]: combined += [msg.to_dict() for msg in self._new_messages[Message.INSTRUCT]] # History context if self._history_messages[Message.CONTEXT]: - combined += [update_dict(msg.to_dict(), 'content', - f"\n{msg.content}\n") - for msg in self._history_messages[Message.CONTEXT]] + combined += [ + update_dict(msg.to_dict(), "content", f"\n{msg.content}\n") + for msg in self._history_messages[Message.CONTEXT] + ] # History chat if self._history_messages[Message.CHAT]: combined += [msg.to_dict() for msg in self._history_messages[Message.CHAT]] @@ -42,9 +44,10 @@ def messages(self) -> List[dict]: combined += [self.request.to_dict()] # New context if self.new_context: - combined += [update_dict(msg.to_dict(), 'content', - f"\n{msg.content}\n") - for msg in self.new_context] + combined += [ + update_dict(msg.to_dict(), "content", f"\n{msg.content}\n") + for msg in self.new_context + ] return combined def input_messages(self, messages: List[dict]): @@ -94,12 +97,13 @@ def input_messages(self, messages: List[dict]): continue self._history_messages[Message.CHAT].append(last_message) - def append_new(self, message_type: str, content: str, - available_tokens: int = sys.maxsize) -> bool: + def append_new( + self, message_type: str, content: str, available_tokens: int = sys.maxsize + ) -> bool: if message_type not in (Message.INSTRUCT, Message.CONTEXT): raise ValueError(f"Current messages cannot be of type {message_type}.") # New instructions and context are of the system role - message = OpenAIMessage(content=content, role='system') + message = OpenAIMessage(content=content, role="system") num_tokens = openai_message_tokens(message.to_dict(), self.model) if num_tokens > available_tokens: @@ -121,8 +125,9 @@ def set_functions(self, functions, available_tokens: int = sys.maxsize): def get_functions(self): return self._new_messages.get(Message.FUNCTION, None) - def _prepend_history(self, message_type: str, message: Message, - token_limit: int = sys.maxsize) -> bool: + def _prepend_history( + self, message_type: str, message: Message, token_limit: int = sys.maxsize + ) -> bool: if message_type == Message.INSTRUCT: raise ValueError("History messages cannot be of type INSTRUCT.") num_tokens = openai_message_tokens(message.to_dict(), self.model) @@ -132,7 +137,7 @@ def _prepend_history(self, message_type: str, message: Message, self._request_tokens += num_tokens return True - def prepend_history(self, prompt: 'OpenAIPrompt', token_limit: int = sys.maxsize) -> bool: + def prepend_history(self, prompt: "OpenAIPrompt", token_limit: int = sys.maxsize) -> bool: # Prepend the first response and the request of the prompt if not self._prepend_history(Message.CHAT, prompt.responses[0], token_limit): return False @@ -148,9 +153,9 @@ def prepend_history(self, prompt: 'OpenAIPrompt', token_limit: int = sys.maxsize def set_request(self, content: str, function_name: Optional[str] = None) -> int: if not content.strip(): raise ValueError("The request cannot be empty.") - message = OpenAIMessage(content=content, - role=('user' if not function_name else 'function'), - name=function_name) + message = OpenAIMessage( + content=content, role=("user" if not function_name else "function"), name=function_name + ) self.request = message self._request_tokens += openai_message_tokens(message.to_dict(), self.model) @@ -166,17 +171,17 @@ def set_response(self, response_str: str): self._timestamp_from_dict(response_data) self._id_from_dict(response_data) - self._request_tokens = response_data['usage']['prompt_tokens'] - self._response_tokens = response_data['usage']['completion_tokens'] + self._request_tokens = response_data["usage"]["prompt_tokens"] + self._response_tokens = response_data["usage"]["completion_tokens"] - for choice in response_data['choices']: - index = choice['index'] + for choice in response_data["choices"]: + index = choice["index"] if index >= len(self.responses): self.responses.extend([None] * (index - len(self.responses) + 1)) self._response_reasons.extend([None] * (index - len(self._response_reasons) + 1)) - self.responses[index] = OpenAIMessage.from_dict(choice['message']) - if choice['finish_reason']: - self._response_reasons[index] = choice['finish_reason'] + self.responses[index] = OpenAIMessage.from_dict(choice["message"]) + if choice["finish_reason"]: + self._response_reasons[index] = choice["finish_reason"] def append_response(self, delta_str: str) -> str: """ @@ -193,11 +198,11 @@ def append_response(self, delta_str: str) -> str: self._timestamp_from_dict(response_data) self._id_from_dict(response_data) - delta_content = '' - for choice in response_data['choices']: - delta = choice['delta'] - index = choice['index'] - finish_reason = choice['finish_reason'] + delta_content = "" + for choice in response_data["choices"]: + delta = choice["delta"] + index = choice["index"] + finish_reason = choice["finish_reason"] if index >= len(self.responses): self.responses.extend([None] * (index - len(self.responses) + 1)) @@ -206,22 +211,24 @@ def append_response(self, delta_str: str) -> str: if not self.responses[index]: self.responses[index] = OpenAIMessage.from_dict(delta) if index == 0: - delta_content = self.responses[0].content if self.responses[0].content else '' + delta_content = self.responses[0].content if self.responses[0].content else "" else: if index == 0: delta_content = self.responses[0].stream_from_dict(delta) else: self.responses[index].stream_from_dict(delta) - if 'function_call' in delta: - if 'name' in delta['function_call'] and \ - self.responses[index].function_call.get('name', '') == '': - self.responses[index].function_call['name'] = \ - delta['function_call']['name'] - if 'arguments' in delta['function_call']: - self.responses[index].function_call['arguments'] = \ - self.responses[index].function_call.get('arguments', '') + \ - delta['function_call']['arguments'] + if "function_call" in delta: + if ( + "name" in delta["function_call"] + and self.responses[index].function_call.get("name", "") == "" + ): + self.responses[index].function_call["name"] = delta["function_call"]["name"] + if "arguments" in delta["function_call"]: + self.responses[index].function_call["arguments"] = ( + self.responses[index].function_call.get("arguments", "") + + delta["function_call"]["arguments"] + ) if finish_reason: self._response_reasons[index] = finish_reason @@ -231,19 +238,19 @@ def _count_response_tokens(self) -> int: return sum(openai_response_tokens(resp.to_dict(), self.model) for resp in self.responses) def _validate_model(self, response_data: dict): - if not response_data['model'].startswith(self.model): - logger.warning("Model mismatch: expected '%s', got '%s'", - self.model, response_data['model']) + if not response_data["model"].startswith(self.model): + logger.warning( + "Model mismatch: expected '%s', got '%s'", self.model, response_data["model"] + ) def _timestamp_from_dict(self, response_data: dict): if not self._timestamp: - self._timestamp = response_data['created'] - elif self._timestamp != response_data['created']: - self._timestamp = response_data['created'] + self._timestamp = response_data["created"] + elif self._timestamp != response_data["created"]: + self._timestamp = response_data["created"] def _id_from_dict(self, response_data: dict): if self._id is None: - self._id = response_data['id'] - elif self._id != response_data['id']: - raise ValueError(f"ID mismatch: expected {self._id}, " - f"got {response_data['id']}") + self._id = response_data["id"] + elif self._id != response_data["id"]: + raise ValueError(f"ID mismatch: expected {self._id}, " f"got {response_data['id']}") diff --git a/devchat/prompt.py b/devchat/prompt.py index 6592e076..e88cf27f 100644 --- a/devchat/prompt.py +++ b/devchat/prompt.py @@ -1,12 +1,12 @@ -from abc import ABC, abstractmethod -from dataclasses import dataclass, field, asdict import hashlib -from datetime import datetime import sys +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, field +from datetime import datetime from typing import Dict, List -from devchat.message import Message -from devchat.utils import unix_to_local_datetime, get_logger, user_id +from devchat.message import Message +from devchat.utils import get_logger, unix_to_local_datetime, user_id logger = get_logger(__name__) @@ -33,16 +33,17 @@ class Prompt(ABC): model: str user_name: str user_email: str - _new_messages: Dict = field(default_factory=lambda: { - Message.INSTRUCT: [], - 'request': None, - Message.CONTEXT: [], - 'responses': [] - }) - _history_messages: Dict[str, Message] = field(default_factory=lambda: { - Message.CONTEXT: [], - Message.CHAT: [] - }) + _new_messages: Dict = field( + default_factory=lambda: { + Message.INSTRUCT: [], + "request": None, + Message.CONTEXT: [], + "responses": [], + } + ) + _history_messages: Dict[str, Message] = field( + default_factory=lambda: {Message.CONTEXT: [], Message.CHAT: []} + ) parent: str = None references: List[str] = field(default_factory=list) _timestamp: int = 0 @@ -59,8 +60,9 @@ def _complete_for_hashing(self) -> bool: bool: Whether the prompt is complete. """ if not self.request or not self.responses: - logger.warning("Incomplete prompt: request = %s, response = %s", - self.request, self.responses) + logger.warning( + "Incomplete prompt: request = %s, response = %s", self.request, self.responses + ) return False if not self.timestamp: @@ -78,15 +80,15 @@ def new_context(self) -> List[Message]: @property def request(self) -> Message: - return self._new_messages['request'] + return self._new_messages["request"] @request.setter def request(self, value: Message): - self._new_messages['request'] = value + self._new_messages["request"] = value @property def responses(self) -> List[Message]: - return self._new_messages['responses'] + return self._new_messages["responses"] @property def timestamp(self) -> int: @@ -142,8 +144,9 @@ def input_messages(self, messages: List[dict]): """ @abstractmethod - def append_new(self, message_type: str, content: str, - available_tokens: int = sys.maxsize) -> bool: + def append_new( + self, message_type: str, content: str, available_tokens: int = sys.maxsize + ) -> bool: """ Append a new message provided by the user to this prompt. @@ -215,9 +218,9 @@ def finalize_hash(self) -> str: self._response_tokens = self._count_response_tokens() data = asdict(self) - data.pop('_hash') + data.pop("_hash") string = str(tuple(sorted(data.items()))) - self._hash = hashlib.sha256(string.encode('utf-8')).hexdigest() + self._hash = hashlib.sha256(string.encode("utf-8")).hexdigest() return self._hash def formatted_header(self) -> str: @@ -238,12 +241,12 @@ def formatted_footer(self, index: int) -> str: note = None formatted_str = "\n\n" reason = self._response_reasons[index] - if reason == 'length': + if reason == "length": note = "Incomplete model output due to max_tokens parameter or token limit" - elif reason == 'function_call': + elif reason == "function_call": formatted_str += self.responses[index].function_call_to_json() + "\n\n" note = "The model decided to call a function" - elif reason == 'content_filter': + elif reason == "content_filter": note = "Omitted content due to a flag from our content filters" if note: @@ -262,8 +265,12 @@ def formatted_full_response(self, index: int) -> str: str: The formatted response string. None if the response is invalid. """ if index >= len(self.responses) or not self.responses[index]: - logger.error("Response index %d is invalid to format: request = %s, response = %s", - index, self.request, self.responses) + logger.error( + "Response index %d is invalid to format: request = %s, response = %s", + index, + self.request, + self.responses, + ) return None formatted_str = "" @@ -280,8 +287,9 @@ def shortlog(self) -> List[dict]: responses = [] for message in self.responses: - responses.append((message.content if message.content else "") - + message.function_call_to_json()) + responses.append( + (message.content if message.content else "") + message.function_call_to_json() + ) return { "user": user_id(self.user_name, self.user_email)[0], @@ -292,5 +300,5 @@ def shortlog(self) -> List[dict]: "request_tokens": self._request_tokens, "response_tokens": self._response_tokens, "hash": self.hash, - "parent": self.parent + "parent": self.parent, } diff --git a/devchat/store.py b/devchat/store.py index 7e40a209..5a18a7b4 100644 --- a/devchat/store.py +++ b/devchat/store.py @@ -1,10 +1,11 @@ -# pylint: disable=import-outside-toplevel -from dataclasses import asdict import json import os -from typing import List, Dict, Any, Optional -from tinydb import TinyDB, where, Query +from dataclasses import asdict +from typing import Any, Dict, List, Optional + +from tinydb import Query, TinyDB, where from tinydb.table import Table + from devchat.chat import Chat from devchat.prompt import Prompt from devchat.utils import get_logger @@ -25,22 +26,24 @@ def __init__(self, store_dir: str, chat: Chat): if not os.path.isdir(store_dir): os.makedirs(store_dir) - self._graph_path = os.path.join(store_dir, 'prompts.graphml') - self._chat_list_path = os.path.join(store_dir, 'prompts_list.json') - self._db_path = os.path.join(store_dir, 'prompts.json') + self._graph_path = os.path.join(store_dir, "prompts.graphml") + self._chat_list_path = os.path.join(store_dir, "prompts_list.json") + self._db_path = os.path.join(store_dir, "prompts.json") self._chat = chat self._db = TinyDB(self._db_path) self._db_meta = self._migrate_db() - self._topics_table = self._db.table('topics') + self._topics_table = self._db.table("topics") if os.path.isfile(self._chat_list_path): - with open(self._chat_list_path, 'r', encoding="utf-8") as file: + with open(self._chat_list_path, "r", encoding="utf-8") as file: self._chat_lists = json.loads(file.read()) elif os.path.isfile(self._graph_path): # convert old graphml to new json from xml.etree.ElementTree import ParseError + import networkx as nx + try: graph = nx.read_graphml(self._graph_path) @@ -48,25 +51,25 @@ def __init__(self, store_dir: str, chat: Chat): self._chat_lists = [] for root in roots: - chat_list = [(root, graph.nodes[root]['timestamp'])] + chat_list = [(root, graph.nodes[root]["timestamp"])] ancestors = nx.ancestors(graph, root) for ancestor in ancestors: - chat_list.append((ancestor, graph.nodes[ancestor]['timestamp'])) + chat_list.append((ancestor, graph.nodes[ancestor]["timestamp"])) self._chat_lists.append(chat_list) - with open(self._chat_list_path, 'w', encoding="utf-8") as file: + with open(self._chat_list_path, "w", encoding="utf-8") as file: file.write(json.dumps(self._chat_lists)) # rename graphml to json - os.rename(self._graph_path, self._graph_path + '.bak') + os.rename(self._graph_path, self._graph_path + ".bak") # update topic table, add request and response fields # new fields: user, date, request, responses, hash visible_topics = self._topics_table.all() for topic in visible_topics: - prompt = self.get_prompt(topic['root']) + prompt = self.get_prompt(topic["root"]) if not prompt: continue self._update_topic_fields(topic, prompt) @@ -81,37 +84,38 @@ def __init__(self, store_dir: str, chat: Chat): self._initialize_topics_table() def _update_topic_fields(self, topic, prompt): - topic['user'] = prompt.user_name - topic['date'] = prompt.timestamp - topic['request'] = prompt.request.content - topic['responses'] = prompt.responses[0].content if prompt.responses else "" - topic['hash'] = prompt.hash - if len(topic['request']) > 100: - topic['request'] = topic['request'][:100] + "..." - if len(topic['responses']) > 100: - topic['responses'] = topic['responses'][:100] + "..." - + topic["user"] = prompt.user_name + topic["date"] = prompt.timestamp + topic["request"] = prompt.request.content + topic["responses"] = prompt.responses[0].content if prompt.responses else "" + topic["hash"] = prompt.hash + if len(topic["request"]) > 100: + topic["request"] = topic["request"][:100] + "..." + if len(topic["responses"]) > 100: + topic["responses"] = topic["responses"][:100] + "..." def _migrate_db(self) -> Table: """ Migrate the database to the latest version. """ - metadata = self._db.table('metadata') + metadata = self._db.table("metadata") + + result = metadata.get(where("version").exists()) + if not result or result["version"].startswith("0.1."): - result = metadata.get(where('version').exists()) - if not result or result['version'].startswith('0.1.'): def replace_response(): def transform(doc): - if '_new_messages' not in doc or 'response' not in doc['_new_messages']: - logger.error("Prompt %s does not match '_new_messages.response'", - doc['_hash']) - doc['_new_messages']['responses'] = doc['_new_messages'].pop('response') + if "_new_messages" not in doc or "response" not in doc["_new_messages"]: + logger.error( + "Prompt %s does not match '_new_messages.response'", doc["_hash"] + ) + doc["_new_messages"]["responses"] = doc["_new_messages"].pop("response") + return transform logger.info("Migrating database from %s to 0.2.0", result) - self._db.update(replace_response(), - Query()._new_messages.response.exists()) # pylint: disable=W0212 - metadata.insert({'version': '0.2.0'}) + self._db.update(replace_response(), Query()._new_messages.response.exists()) + metadata.insert({"version": "0.2.0"}) return metadata def _initialize_topics_table(self): @@ -120,18 +124,13 @@ def _initialize_topics_table(self): continue first = chat_list[0] - last = chat_list[-1] + last = chat_list[-1] - topic = { - 'root': first[0], - 'latest_time': last[1], - 'title': None, - 'hidden': False - } + topic = {"root": first[0], "latest_time": last[1], "title": None, "hidden": False} - prompt = self.get_prompt(topic['root']) + prompt = self.get_prompt(topic["root"]) if not prompt: - logger.error("Prompt %s not found while selecting from the store", topic['root']) + logger.error("Prompt %s not found while selecting from the store", topic["root"]) continue self._update_topic_fields(topic, prompt) @@ -145,17 +144,17 @@ def _update_topics_table(self, prompt: Prompt): if chat_list[-1][0] == prompt.hash: topic_hash = chat_list[0][0] - topic = next((t for t in self._topics_table if t['root'] == topic_hash), None) + topic = next((t for t in self._topics_table if t["root"] == topic_hash), None) if topic: - topic['latest_time'] = max(topic.get('latest_time', 0), prompt.timestamp) + topic["latest_time"] = max(topic.get("latest_time", 0), prompt.timestamp) self._topics_table.update(topic, doc_ids=[topic.doc_id]) break else: topic = { - 'root': prompt.hash, - 'latest_time': prompt.timestamp, - 'title': None, - 'hidden': False + "root": prompt.hash, + "latest_time": prompt.timestamp, + "title": None, + "hidden": False, } self._update_topic_fields(topic, prompt) self._topics_table.insert(topic) @@ -187,12 +186,11 @@ def store_prompt(self, prompt: Prompt) -> str: self._chat_lists.append([(prompt.hash, prompt.timestamp)]) self._update_topics_table(prompt) - with open(self._chat_list_path, 'w', encoding="utf-8") as file: + with open(self._chat_list_path, "w", encoding="utf-8") as file: file.write(json.dumps(self._chat_lists)) return topic_hash - def get_prompt(self, prompt_hash: str) -> Prompt: """ Retrieve a prompt from the store. @@ -203,7 +201,7 @@ def get_prompt(self, prompt_hash: str) -> Prompt: Prompt: The retrieved prompt. None if the prompt is not found. """ # Retrieve the prompt object from TinyDB - prompt_data = self._db.search(where('_hash') == prompt_hash) + prompt_data = self._db.search(where("_hash") == prompt_hash) if not prompt_data: logger.warning("Prompt %s not found while retrieving from object store.", prompt_hash) return None @@ -266,24 +264,25 @@ def select_topics(self, start: int, end: int) -> List[Dict[str, Any]]: List[Dict[str, Any]]: A list of dictionaries containing root prompts with latest_time, and title fields. """ - visible_topics = self._topics_table.search( - where('hidden') == False) # pylint: disable=C0121 - sorted_topics = sorted(visible_topics, key=lambda x: x['latest_time'], reverse=True) + visible_topics = self._topics_table.search(where("hidden") == False) # noqa: E712 + sorted_topics = sorted(visible_topics, key=lambda x: x["latest_time"], reverse=True) topics = [] for topic in sorted_topics[start:end]: - topics.append({ - 'root_prompt': { - 'hash': topic['root'], - 'user': topic['user'], - 'date': topic['date'], - 'request': topic['request'], - 'responses': [topic['responses']], - }, - 'latest_time': topic['latest_time'], - 'title': topic['title'], - 'hidden': topic['hidden'], - }) + topics.append( + { + "root_prompt": { + "hash": topic["root"], + "user": topic["user"], + "date": topic["date"], + "request": topic["request"], + "responses": [topic["responses"]], + }, + "latest_time": topic["latest_time"], + "title": topic["title"], + "hidden": topic["hidden"], + } + ) return topics def delete_prompt(self, prompt_hash: str) -> bool: @@ -316,13 +315,13 @@ def delete_prompt(self, prompt_hash: str) -> bool: return False # Update the topics table - self._topics_table.remove(where('root') == prompt_hash) + self._topics_table.remove(where("root") == prompt_hash) # Remove the prompt from the database - self._db.remove(where('_hash') == prompt_hash) + self._db.remove(where("_hash") == prompt_hash) # Save the graph - with open(self._chat_list_path, 'w', encoding="utf-8") as file: + with open(self._chat_list_path, "w", encoding="utf-8") as file: file.write(json.dumps(self._chat_lists)) return True diff --git a/devchat/utils.py b/devchat/utils.py index d6903c60..2c179bfa 100644 --- a/devchat/utils.py +++ b/devchat/utils.py @@ -1,19 +1,17 @@ -# pylint: disable=import-outside-toplevel +import datetime +import getpass +import hashlib import logging import os import re -import getpass import socket import subprocess -from typing import List, Tuple, Optional -import datetime -import hashlib - +from typing import List, Optional, Tuple -log_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') -# pylint: disable=invalid-name +log_formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") encoding = None + def setup_logger(file_path: Optional[str] = None): """Utility function to set up a global file log handler.""" if file_path is None: @@ -28,7 +26,7 @@ def get_logger(name: str = None, handler: logging.Handler = None) -> logging.Log local_logger = logging.getLogger(name) # Default to 'INFO' if 'LOG_LEVEL' env is not set - log_level_str = os.getenv('LOG_LEVEL', 'INFO') + log_level_str = os.getenv("LOG_LEVEL", "INFO") log_level = getattr(logging, log_level_str.upper(), logging.INFO) local_logger.setLevel(log_level) @@ -55,9 +53,13 @@ def find_root_dir() -> Tuple[Optional[str], Optional[str]]: repo_dir = None try: - repo_dir = subprocess.run(["git", "rev-parse", "--show-toplevel"], - capture_output=True, text=True, check=True, - encoding='utf-8').stdout.strip() + repo_dir = subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + capture_output=True, + text=True, + check=True, + encoding="utf-8", + ).stdout.strip() if not os.path.isdir(repo_dir): repo_dir = None else: @@ -66,8 +68,9 @@ def find_root_dir() -> Tuple[Optional[str], Optional[str]]: repo_dir = None try: - result = subprocess.run(["svn", "info"], - capture_output=True, text=True, check=True, encoding='utf-8') + result = subprocess.run( + ["svn", "info"], capture_output=True, text=True, check=True, encoding="utf-8" + ) if result.returncode == 0: for line in result.stdout.splitlines(): if line.startswith("Working Copy Root Path: "): @@ -81,10 +84,10 @@ def find_root_dir() -> Tuple[Optional[str], Optional[str]]: def add_gitignore(target_dir: str, *ignore_entries: str) -> None: - gitignore_path = os.path.join(target_dir, '.gitignore') + gitignore_path = os.path.join(target_dir, ".gitignore") if os.path.exists(gitignore_path): - with open(gitignore_path, 'r', encoding='utf-8') as gitignore_file: + with open(gitignore_path, "r", encoding="utf-8") as gitignore_file: gitignore_content = gitignore_file.read() new_entries = [] @@ -93,15 +96,15 @@ def add_gitignore(target_dir: str, *ignore_entries: str) -> None: new_entries.append(entry) if new_entries: - with open(gitignore_path, 'a', encoding='utf-8') as gitignore_file: - gitignore_file.write('\n# devchat\n') + with open(gitignore_path, "a", encoding="utf-8") as gitignore_file: + gitignore_file.write("\n# devchat\n") for entry in new_entries: - gitignore_file.write(f'{entry}\n') + gitignore_file.write(f"{entry}\n") else: - with open(gitignore_path, 'w', encoding='utf-8') as gitignore_file: - gitignore_file.write('# devchat\n') + with open(gitignore_path, "w", encoding="utf-8") as gitignore_file: + gitignore_file.write("# devchat\n") for entry in ignore_entries: - gitignore_file.write(f'{entry}\n') + gitignore_file.write(f"{entry}\n") def unix_to_local_datetime(unix_time) -> datetime.datetime: @@ -116,8 +119,8 @@ def unix_to_local_datetime(unix_time) -> datetime.datetime: def get_user_info() -> Tuple[str, str]: try: - cmd = ['git', 'config', 'user.name'] - user_name = subprocess.check_output(cmd, encoding='utf-8').strip() + cmd = ["git", "config", "user.name"] + user_name = subprocess.check_output(cmd, encoding="utf-8").strip() except Exception: try: user_name = getpass.getuser() @@ -126,17 +129,17 @@ def get_user_info() -> Tuple[str, str]: user_name = user_dir.split(os.sep)[-1] try: - cmd = ['git', 'config', 'user.email'] - user_email = subprocess.check_output(cmd, encoding='utf-8').strip() + cmd = ["git", "config", "user.email"] + user_email = subprocess.check_output(cmd, encoding="utf-8").strip() except Exception: - user_email = user_name + '@' + socket.gethostname() + user_email = user_name + "@" + socket.gethostname() return user_name, user_email def user_id(user_name, user_email) -> Tuple[str, str]: user_str = f"{user_name} <{user_email}>" - user_hash = hashlib.sha1(user_str.encode('utf-8')).hexdigest() + user_hash = hashlib.sha1(user_str.encode("utf-8")).hexdigest() return user_str, user_hash @@ -151,7 +154,7 @@ def parse_files(file_paths: List[str]) -> List[str]: contents = [] for file_path in file_paths: - with open(file_path, 'r', encoding='utf-8') as file: + with open(file_path, "r", encoding="utf-8") as file: content = file.read() if not content: raise ValueError(f"File {file_path} is empty.") @@ -161,7 +164,7 @@ def parse_files(file_paths: List[str]) -> List[str]: def valid_hash(hash_str): """Check if a string is a valid hash value.""" - pattern = re.compile(r'^[a-f0-9]{64}$') # for SHA-256 hash + pattern = re.compile(r"^[a-f0-9]{64}$") # for SHA-256 hash return bool(pattern.match(hash_str)) @@ -198,25 +201,24 @@ def update_dict(dict_to_update, key, value) -> dict: return dict_to_update -def openai_message_tokens(messages: dict, model: str) -> int: # pylint: disable=unused-argument +def openai_message_tokens(messages: dict, model: str) -> int: """Returns the number of tokens used by a message.""" if not os.environ.get("USE_TIKTOKEN", False): - return len(str(messages))/4 + return len(str(messages)) / 4 - # pylint: disable=global-statement global encoding if not encoding: import tiktoken script_dir = os.path.dirname(os.path.realpath(__file__)) - os.environ['TIKTOKEN_CACHE_DIR'] = os.path.join(script_dir, 'tiktoken_cache') + os.environ["TIKTOKEN_CACHE_DIR"] = os.path.join(script_dir, "tiktoken_cache") try: encoding = tiktoken.get_encoding("cl100k_base") except Exception: from tiktoken import registry - from tiktoken.registry import _find_constructors from tiktoken.core import Encoding + from tiktoken.registry import _find_constructors def get_encoding(name: str): _find_constructors() diff --git a/devchat/workflow/cli.py b/devchat/workflow/cli.py index 90038958..0c5a8f59 100644 --- a/devchat/workflow/cli.py +++ b/devchat/workflow/cli.py @@ -1,8 +1,9 @@ import click -from devchat.workflow.command.update import update -from devchat.workflow.command.list import list_cmd -from devchat.workflow.command.env import env + from devchat.workflow.command.config import config_cmd +from devchat.workflow.command.env import env +from devchat.workflow.command.list import list_cmd +from devchat.workflow.command.update import update @click.group(help="CLI for devchat workflow engine.") diff --git a/devchat/workflow/command/config.py b/devchat/workflow/command/config.py index 60f944f2..6c106a3d 100644 --- a/devchat/workflow/command/config.py +++ b/devchat/workflow/command/config.py @@ -1,6 +1,6 @@ import json - from pathlib import Path + import click import oyaml as yaml @@ -10,7 +10,6 @@ @click.command(help="Workflow configuration.", name="config") @click.option("--json", "in_json", is_flag=True, help="Output in json format.") def config_cmd(in_json: bool): - config_path = Path(WORKFLOWS_BASE) / WORKFLOWS_CONFIG_FILENAME config_content = {} if config_path.exists(): diff --git a/devchat/workflow/command/env.py b/devchat/workflow/command/env.py index e1ce6c30..14c2adfc 100644 --- a/devchat/workflow/command/env.py +++ b/devchat/workflow/command/env.py @@ -1,14 +1,14 @@ -# pylint: disable=invalid-name - """ Commands for managing the python environment of workflows. """ import sys from pathlib import Path -from typing import Optional, List +from typing import List, Optional + import click -from devchat.workflow.env_manager import PyEnvManager, MAMBA_PY_ENVS + +from devchat.workflow.env_manager import MAMBA_PY_ENVS, PyEnvManager def _get_all_env_names() -> List[str]: @@ -19,11 +19,7 @@ def _get_all_env_names() -> List[str]: excludes = ["devchat", "devchat-ask", "devchat-commands"] envs_path = Path(MAMBA_PY_ENVS) - envs = [ - env.name - for env in envs_path.iterdir() - if env.is_dir() and env.name not in excludes - ] + envs = [env.name for env in envs_path.iterdir() if env.is_dir() and env.name not in excludes] return envs @@ -42,9 +38,7 @@ def list_envs(): required=False, type=str, ) -@click.option( - "--all", "all_flag", help="Remove all the python envs of workflows.", is_flag=True -) +@click.option("--all", "all_flag", help="Remove all the python envs of workflows.", is_flag=True) def remove(env_name: Optional[str] = None, all_flag: bool = False): if not env_name and not all_flag: click.echo("Please provide the name of the python env to remove.") diff --git a/devchat/workflow/command/list.py b/devchat/workflow/command/list.py index 84f01ebb..e6fd664c 100644 --- a/devchat/workflow/command/list.py +++ b/devchat/workflow/command/list.py @@ -1,19 +1,19 @@ import json +from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import List, Set, Tuple, Dict -from dataclasses import dataclass, asdict, field +from typing import Dict, List, Set, Tuple import click import oyaml as yaml import yaml as pyyaml +from devchat.utils import get_logger from devchat.workflow.namespace import get_prioritized_namespace_path from devchat.workflow.path import COMMAND_FILENAMES -from devchat.utils import get_logger - logger = get_logger(__name__) + @dataclass class WorkflowMeta: name: str @@ -27,9 +27,7 @@ def __str__(self): return f"{'*' if self.active else ' '} {self.name} ({self.namespace})" -def iter_namespace( - ns_path: str, existing_names: Set[str] -) -> Tuple[List[WorkflowMeta], Set[str]]: +def iter_namespace(ns_path: str, existing_names: Set[str]) -> Tuple[List[WorkflowMeta], Set[str]]: """ Get all workflows under the namespace path. diff --git a/devchat/workflow/command/update.py b/devchat/workflow/command/update.py index 4629c3e6..fba3da51 100644 --- a/devchat/workflow/command/update.py +++ b/devchat/workflow/command/update.py @@ -1,27 +1,25 @@ -# pylint: disable=invalid-name - import os import shutil import tempfile import zipfile -from typing import List, Optional, Tuple -from pathlib import Path from datetime import datetime +from pathlib import Path +from typing import List, Optional, Tuple import click import requests +from devchat.utils import get_logger from devchat.workflow.path import ( CHAT_DIR, + CUSTOM_BASE, WORKFLOWS_BASE, WORKFLOWS_BASE_NAME, - CUSTOM_BASE, ) -from devchat.utils import get_logger HAS_GIT = False try: - from git import Repo, InvalidGitRepositoryError, GitCommandError + from git import GitCommandError, InvalidGitRepositoryError, Repo except ImportError: pass else: @@ -254,9 +252,7 @@ def update_by_git(workflow_base: Path): remote_main_hash = repo.commit(f"origin/{DEFAULT_BRANCH}").hexsha if local_main_hash == remote_main_hash: - click.echo( - f"Local branch is up-to-date with remote {DEFAULT_BRANCH}. Skip update." - ) + click.echo(f"Local branch is up-to-date with remote {DEFAULT_BRANCH}. Skip update.") return try: @@ -290,15 +286,11 @@ def copy_workflows_usr(): shutil.copytree(old_usr_dir, new_usr_dir) click.echo(f"Copied {old_usr_dir} to {new_usr_dir} successfully.") else: - click.echo( - f"Skip copying usr dir. old exists: {old_exists}, new exists: {new_exists}." - ) + click.echo(f"Skip copying usr dir. old exists: {old_exists}, new exists: {new_exists}.") @click.command(help="Update the workflow_base dir.") -@click.option( - "-f", "--force", is_flag=True, help="Force update the workflows to the latest main." -) +@click.option("-f", "--force", is_flag=True, help="Force update the workflows to the latest main.") def update(force: bool): click.echo(f"Updating wf repo... force: {force}") click.echo(f"WORKFLOWS_BASE: {WORKFLOWS_BASE}") diff --git a/devchat/workflow/env_manager.py b/devchat/workflow/env_manager.py index 7bfe8c1e..36e5223d 100644 --- a/devchat/workflow/env_manager.py +++ b/devchat/workflow/env_manager.py @@ -1,14 +1,12 @@ -# pylint: disable=invalid-name import os -import sys import subprocess -from typing import Optional, Dict +import sys +from typing import Dict, Optional from .envs import MAMBA_BIN_PATH from .path import MAMBA_PY_ENVS, MAMBA_ROOT -from .user_setting import USER_SETTINGS from .schema import ExternalPyConf - +from .user_setting import USER_SETTINGS # CONDA_FORGE = [ # "https://mirrors.tuna.tsinghua.edu.cn/anaconda/cloud/conda-forge/", @@ -28,8 +26,10 @@ def _get_external_envs() -> Dict[str, ExternalPyConf]: return external_pythons + EXTERNAL_ENVS = _get_external_envs() + class PyEnvManager: mamba_bin = MAMBA_BIN_PATH mamba_root = MAMBA_ROOT @@ -82,15 +82,11 @@ def install(self, env_name: str, requirements_file: str) -> bool: ] env = os.environ.copy() env.pop("PYTHONPATH") - with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env - ) as proc: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) as proc: proc.wait() if proc.returncode != 0: - print( - f"Failed to install requirements: {requirements_file}", flush=True - ) + print(f"Failed to install requirements: {requirements_file}", flush=True) return False return True @@ -149,9 +145,7 @@ def create(self, env_name: str, py_version: str) -> bool: f"python={py_version}", "-y", ] - with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) as proc: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: proc.wait() if proc.returncode != 0: @@ -178,9 +172,7 @@ def remove(self, env_name: str) -> bool: self.mamba_root, "-y", ] - with subprocess.Popen( - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) as proc: + with subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: proc.wait() if proc.returncode != 0: diff --git a/devchat/workflow/envs.py b/devchat/workflow/envs.py index f50b6997..67aa0639 100644 --- a/devchat/workflow/envs.py +++ b/devchat/workflow/envs.py @@ -1,6 +1,7 @@ """ Explicitly define the environment variables used in the workflow engine. """ + import os PYTHON_PATH = os.environ.get("PYTHONPATH", "") diff --git a/devchat/workflow/namespace.py b/devchat/workflow/namespace.py index 8802bde6..306ca04c 100644 --- a/devchat/workflow/namespace.py +++ b/devchat/workflow/namespace.py @@ -4,15 +4,17 @@ import os from typing import List -from pydantic import BaseModel, Extra, ValidationError + import oyaml as yaml +from pydantic import BaseModel, Extra, ValidationError + from devchat.utils import get_logger from .path import ( - CUSTOM_BASE, - MERICO_WORKFLOWS, COMMUNITY_WORKFLOWS, + CUSTOM_BASE, CUSTOM_CONFIG_FILE, + MERICO_WORKFLOWS, ) logger = get_logger(__name__) diff --git a/devchat/workflow/schema.py b/devchat/workflow/schema.py index 4b27a699..5be1fc53 100644 --- a/devchat/workflow/schema.py +++ b/devchat/workflow/schema.py @@ -1,7 +1,7 @@ import re -from typing import Optional, List, Dict, Union +from typing import Dict, List, Optional, Union -from pydantic import BaseModel, validator, Extra, ValidationError +from pydantic import BaseModel, Extra, ValidationError, validator class WorkflowPyConf(BaseModel): @@ -10,7 +10,7 @@ class WorkflowPyConf(BaseModel): env_name: Optional[str] # python env name, will use the workflow name if not set @validator("version") - def validate_version(cls, value): # pylint: disable=no-self-argument + def validate_version(cls, value): pattern = r"^\d+\.\d+(\.\d+)?$" if not re.match(pattern, value): raise ValidationError( @@ -41,7 +41,7 @@ class WorkflowConfig(BaseModel): help: Optional[Union[str, Dict[str, str]]] = None @validator("input_required", pre=True) - def to_boolean(cls, value): # pylint: disable=no-self-argument + def to_boolean(cls, value): return value.lower() == "required" class Config: diff --git a/devchat/workflow/step.py b/devchat/workflow/step.py index fe6b6777..93cc97a4 100644 --- a/devchat/workflow/step.py +++ b/devchat/workflow/step.py @@ -1,15 +1,14 @@ -# pylint: disable=invalid-name - +import json import os +import shlex +import subprocess import sys import threading -import subprocess -import shlex -import json -from typing import Dict, Tuple, List from enum import Enum -from .schema import WorkflowConfig, RuntimeParameter +from typing import Dict, List, Tuple + from .path import WORKFLOWS_BASE +from .schema import RuntimeParameter, WorkflowConfig class BuiltInVars(str, Enum): @@ -47,9 +46,7 @@ def command_raw(self) -> str: """ return self._kwargs.get("run", "") - def _setup_env( - self, wf_config: WorkflowConfig, rt_param: RuntimeParameter - ) -> Dict[str, str]: + def _setup_env(self, wf_config: WorkflowConfig, rt_param: RuntimeParameter) -> Dict[str, str]: """ Setup the environment variables for the subprocess. """ @@ -96,8 +93,7 @@ def _validate_and_interpolate( if BuiltInVars.workflow_python in command_raw: if not rt_param.workflow_python: raise ValueError( - "The command uses $workflow_python, " - "but the workflow_python is not set yet." + "The command uses $workflow_python, " "but the workflow_python is not set yet." ) args = [] @@ -129,10 +125,7 @@ def _validate_and_interpolate( return args - - def run( - self, wf_config: WorkflowConfig, rt_param: RuntimeParameter - ) -> Tuple[int, str, str]: + def run(self, wf_config: WorkflowConfig, rt_param: RuntimeParameter) -> Tuple[int, str, str]: """ Run the step in a subprocess. diff --git a/devchat/workflow/user_setting.py b/devchat/workflow/user_setting.py index 60ec00ca..5574668f 100644 --- a/devchat/workflow/user_setting.py +++ b/devchat/workflow/user_setting.py @@ -1,7 +1,9 @@ from pathlib import Path + import oyaml as yaml + +from .path import USER_SETTINGS_FILENAME, WORKFLOWS_BASE from .schema import UserSettings -from .path import WORKFLOWS_BASE, USER_SETTINGS_FILENAME def _load_user_settings() -> UserSettings: @@ -20,4 +22,5 @@ def _load_user_settings() -> UserSettings: return UserSettings() + USER_SETTINGS = _load_user_settings() diff --git a/devchat/workflow/workflow.py b/devchat/workflow/workflow.py index ba327bd3..d9f8925e 100644 --- a/devchat/workflow/workflow.py +++ b/devchat/workflow/workflow.py @@ -1,15 +1,14 @@ -# pylint: disable=invalid-name - import os import sys -from typing import Optional, Tuple, List, Dict +from typing import Dict, List, Optional, Tuple + import oyaml as yaml -from .step import WorkflowStep -from .schema import WorkflowConfig, RuntimeParameter -from .path import COMMAND_FILENAMES -from .namespace import get_prioritized_namespace_path -from .env_manager import PyEnvManager, EXTERNAL_ENVS +from .env_manager import EXTERNAL_ENVS, PyEnvManager +from .namespace import get_prioritized_namespace_path +from .path import COMMAND_FILENAMES +from .schema import RuntimeParameter, WorkflowConfig +from .step import WorkflowStep class Workflow: @@ -47,9 +46,7 @@ def parse_trigger(user_input: str) -> Tuple[Optional[str], Optional[str]]: workflow_name = striped.split()[0][1:] # remove the trigger prefix and the workflow name - actual_input = user_input.replace( - f"{Workflow.TRIGGER_PREFIX}{workflow_name}", "", 1 - ) + actual_input = user_input.replace(f"{Workflow.TRIGGER_PREFIX}{workflow_name}", "", 1) return workflow_name, actual_input @staticmethod diff --git a/poetry.lock b/poetry.lock index 10cd49e0..934e71a5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -22,25 +22,6 @@ doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphin test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] trio = ["trio (>=0.23)"] -[[package]] -name = "astroid" -version = "2.15.8" -description = "An abstract syntax tree for Python with inference support." -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "astroid-2.15.8-py3-none-any.whl", hash = "sha256:1aa149fc5c6589e3d0ece885b4491acd80af4f087baafa3fb5203b113e68cd3c"}, - {file = "astroid-2.15.8.tar.gz", hash = "sha256:6c107453dffee9055899705de3c9ead36e74119cee151e5a9aaf7f0b0e020a6a"}, -] - -[package.dependencies] -lazy-object-proxy = ">=1.4.0" -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} -wrapt = [ - {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, -] - [[package]] name = "certifi" version = "2024.2.2" @@ -176,21 +157,6 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] -[[package]] -name = "dill" -version = "0.3.8" -description = "serialize all of Python" -optional = false -python-versions = ">=3.8" -files = [ - {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, - {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] -profile = ["gprof2dot (>=2022.7.29)"] - [[package]] name = "distro" version = "1.9.0" @@ -362,66 +328,6 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] -[[package]] -name = "isort" -version = "5.13.2" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, - {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, -] - -[package.extras] -colors = ["colorama (>=0.4.6)"] - -[[package]] -name = "lazy-object-proxy" -version = "1.10.0" -description = "A fast and thorough lazy object proxy." -optional = false -python-versions = ">=3.8" -files = [ - {file = "lazy-object-proxy-1.10.0.tar.gz", hash = "sha256:78247b6d45f43a52ef35c25b5581459e85117225408a4128a3daf8bf9648ac69"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:855e068b0358ab916454464a884779c7ffa312b8925c6f7401e952dcf3b89977"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab7004cf2e59f7c2e4345604a3e6ea0d92ac44e1c2375527d56492014e690c3"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc0d2fc424e54c70c4bc06787e4072c4f3b1aa2f897dfdc34ce1013cf3ceef05"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e2adb09778797da09d2b5ebdbceebf7dd32e2c96f79da9052b2e87b6ea495895"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b1f711e2c6dcd4edd372cf5dec5c5a30d23bba06ee012093267b3376c079ec83"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-win32.whl", hash = "sha256:76a095cfe6045c7d0ca77db9934e8f7b71b14645f0094ffcd842349ada5c5fb9"}, - {file = "lazy_object_proxy-1.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:b4f87d4ed9064b2628da63830986c3d2dca7501e6018347798313fcf028e2fd4"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fec03caabbc6b59ea4a638bee5fce7117be8e99a4103d9d5ad77f15d6f81020c"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:02c83f957782cbbe8136bee26416686a6ae998c7b6191711a04da776dc9e47d4"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:009e6bb1f1935a62889ddc8541514b6a9e1fcf302667dcb049a0be5c8f613e56"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:75fc59fc450050b1b3c203c35020bc41bd2695ed692a392924c6ce180c6f1dc9"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:782e2c9b2aab1708ffb07d4bf377d12901d7a1d99e5e410d648d892f8967ab1f"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-win32.whl", hash = "sha256:edb45bb8278574710e68a6b021599a10ce730d156e5b254941754a9cc0b17d03"}, - {file = "lazy_object_proxy-1.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:e271058822765ad5e3bca7f05f2ace0de58a3f4e62045a8c90a0dfd2f8ad8cc6"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e98c8af98d5707dcdecc9ab0863c0ea6e88545d42ca7c3feffb6b4d1e370c7ba"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:952c81d415b9b80ea261d2372d2a4a2332a3890c2b83e0535f263ddfe43f0d43"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80b39d3a151309efc8cc48675918891b865bdf742a8616a337cb0090791a0de9"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:e221060b701e2aa2ea991542900dd13907a5c90fa80e199dbf5a03359019e7a3"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92f09ff65ecff3108e56526f9e2481b8116c0b9e1425325e13245abfd79bdb1b"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-win32.whl", hash = "sha256:3ad54b9ddbe20ae9f7c1b29e52f123120772b06dbb18ec6be9101369d63a4074"}, - {file = "lazy_object_proxy-1.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:127a789c75151db6af398b8972178afe6bda7d6f68730c057fbbc2e96b08d282"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4ed0518a14dd26092614412936920ad081a424bdcb54cc13349a8e2c6d106a"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ad9e6ed739285919aa9661a5bbed0aaf410aa60231373c5579c6b4801bd883c"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fc0a92c02fa1ca1e84fc60fa258458e5bf89d90a1ddaeb8ed9cc3147f417255"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0aefc7591920bbd360d57ea03c995cebc204b424524a5bd78406f6e1b8b2a5d8"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5faf03a7d8942bb4476e3b62fd0f4cf94eaf4618e304a19865abf89a35c0bbee"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-win32.whl", hash = "sha256:e333e2324307a7b5d86adfa835bb500ee70bfcd1447384a822e96495796b0ca4"}, - {file = "lazy_object_proxy-1.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:cb73507defd385b7705c599a94474b1d5222a508e502553ef94114a143ec6696"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:366c32fe5355ef5fc8a232c5436f4cc66e9d3e8967c01fb2e6302fd6627e3d94"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2297f08f08a2bb0d32a4265e98a006643cd7233fb7983032bd61ac7a02956b3b"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18dd842b49456aaa9a7cf535b04ca4571a302ff72ed8740d06b5adcd41fe0757"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:217138197c170a2a74ca0e05bddcd5f1796c735c37d0eee33e43259b192aa424"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9a3a87cf1e133e5b1994144c12ca4aa3d9698517fe1e2ca82977781b16955658"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-win32.whl", hash = "sha256:30b339b2a743c5288405aa79a69e706a06e02958eab31859f7f3c04980853b70"}, - {file = "lazy_object_proxy-1.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:a899b10e17743683b293a729d3a11f2f399e8a90c73b089e29f5d0fe3509f0dd"}, - {file = "lazy_object_proxy-1.10.0-pp310.pp311.pp312.pp38.pp39-none-any.whl", hash = "sha256:80fa48bd89c8f2f456fc0765c11c23bf5af827febacd2f523ca5bc1893fcc09d"}, -] - [[package]] name = "markdown-it-py" version = "3.0.0" @@ -446,17 +352,6 @@ profiling = ["gprof2dot"] rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -545,21 +440,6 @@ files = [ {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] -[[package]] -name = "platformdirs" -version = "4.2.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.8" -files = [ - {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, - {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, -] - -[package.extras] -docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] - [[package]] name = "pluggy" version = "1.4.0" @@ -642,35 +522,6 @@ files = [ plugins = ["importlib-metadata"] windows-terminal = ["colorama (>=0.4.6)"] -[[package]] -name = "pylint" -version = "2.17.7" -description = "python code static checker" -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "pylint-2.17.7-py3-none-any.whl", hash = "sha256:27a8d4c7ddc8c2f8c18aa0050148f89ffc09838142193fdbe98f172781a3ff87"}, - {file = "pylint-2.17.7.tar.gz", hash = "sha256:f4fcac7ae74cfe36bc8451e931d8438e4a476c20314b1101c458ad0f05191fad"}, -] - -[package.dependencies] -astroid = ">=2.15.8,<=2.17.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, -] -isort = ">=4.2.5,<6" -mccabe = ">=0.6,<0.8" -platformdirs = ">=2.2.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} - -[package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] - [[package]] name = "pytest" version = "7.4.4" @@ -705,7 +556,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -713,15 +563,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -738,7 +581,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -746,7 +588,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -913,6 +754,32 @@ typing-extensions = "*" [package.extras] dev = ["flake8", "flake8-docstrings", "mypy", "packaging", "pre-commit", "pytest", "pytest-cov", "types-setuptools"] +[[package]] +name = "ruff" +version = "0.4.4" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.4.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:29d44ef5bb6a08e235c8249294fa8d431adc1426bfda99ed493119e6f9ea1bf6"}, + {file = "ruff-0.4.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c4efe62b5bbb24178c950732ddd40712b878a9b96b1d02b0ff0b08a090cbd891"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c8e2f1e8fc12d07ab521a9005d68a969e167b589cbcaee354cb61e9d9de9c15"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:60ed88b636a463214905c002fa3eaab19795679ed55529f91e488db3fe8976ab"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b90fc5e170fc71c712cc4d9ab0e24ea505c6a9e4ebf346787a67e691dfb72e85"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:8e7e6ebc10ef16dcdc77fd5557ee60647512b400e4a60bdc4849468f076f6eef"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b9ddb2c494fb79fc208cd15ffe08f32b7682519e067413dbaf5f4b01a6087bcd"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c51c928a14f9f0a871082603e25a1588059b7e08a920f2f9fa7157b5bf08cfe9"}, + {file = "ruff-0.4.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5eb0a4bfd6400b7d07c09a7725e1a98c3b838be557fee229ac0f84d9aa49c36"}, + {file = "ruff-0.4.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b1867ee9bf3acc21778dcb293db504692eda5f7a11a6e6cc40890182a9f9e595"}, + {file = "ruff-0.4.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1aecced1269481ef2894cc495647392a34b0bf3e28ff53ed95a385b13aa45768"}, + {file = "ruff-0.4.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9da73eb616b3241a307b837f32756dc20a0b07e2bcb694fec73699c93d04a69e"}, + {file = "ruff-0.4.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:958b4ea5589706a81065e2a776237de2ecc3e763342e5cc8e02a4a4d8a5e6f95"}, + {file = "ruff-0.4.4-py3-none-win32.whl", hash = "sha256:cb53473849f011bca6e754f2cdf47cafc9c4f4ff4570003a0dad0b9b6890e876"}, + {file = "ruff-0.4.4-py3-none-win_amd64.whl", hash = "sha256:424e5b72597482543b684c11def82669cc6b395aa8cc69acc1858b5ef3e5daae"}, + {file = "ruff-0.4.4-py3-none-win_arm64.whl", hash = "sha256:39df0537b47d3b597293edbb95baf54ff5b49589eb7ff41926d8243caa995ea6"}, + {file = "ruff-0.4.4.tar.gz", hash = "sha256:f87ea42d5cdebdc6a69761a9d0bc83ae9b3b30d0ad78952005ba6568d6c022af"}, +] + [[package]] name = "smmap" version = "5.0.1" @@ -1016,17 +883,6 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] -[[package]] -name = "tomlkit" -version = "0.12.4" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomlkit-0.12.4-py3-none-any.whl", hash = "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b"}, - {file = "tomlkit-0.12.4.tar.gz", hash = "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3"}, -] - [[package]] name = "tqdm" version = "4.66.2" @@ -1074,85 +930,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "wrapt" -version = "1.16.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = ">=3.6" -files = [ - {file = "wrapt-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ffa565331890b90056c01db69c0fe634a776f8019c143a5ae265f9c6bc4bd6d4"}, - {file = "wrapt-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e4fdb9275308292e880dcbeb12546df7f3e0f96c6b41197e0cf37d2826359020"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb2dee3874a500de01c93d5c71415fcaef1d858370d405824783e7a8ef5db440"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a88e6010048489cda82b1326889ec075a8c856c2e6a256072b28eaee3ccf487"}, - {file = "wrapt-1.16.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac83a914ebaf589b69f7d0a1277602ff494e21f4c2f743313414378f8f50a4cf"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:73aa7d98215d39b8455f103de64391cb79dfcad601701a3aa0dddacf74911d72"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:807cc8543a477ab7422f1120a217054f958a66ef7314f76dd9e77d3f02cdccd0"}, - {file = "wrapt-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bf5703fdeb350e36885f2875d853ce13172ae281c56e509f4e6eca049bdfb136"}, - {file = "wrapt-1.16.0-cp310-cp310-win32.whl", hash = "sha256:f6b2d0c6703c988d334f297aa5df18c45e97b0af3679bb75059e0e0bd8b1069d"}, - {file = "wrapt-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:decbfa2f618fa8ed81c95ee18a387ff973143c656ef800c9f24fb7e9c16054e2"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a5db485fe2de4403f13fafdc231b0dbae5eca4359232d2efc79025527375b09"}, - {file = "wrapt-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75ea7d0ee2a15733684badb16de6794894ed9c55aa5e9903260922f0482e687d"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a452f9ca3e3267cd4d0fcf2edd0d035b1934ac2bd7e0e57ac91ad6b95c0c6389"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43aa59eadec7890d9958748db829df269f0368521ba6dc68cc172d5d03ed8060"}, - {file = "wrapt-1.16.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:72554a23c78a8e7aa02abbd699d129eead8b147a23c56e08d08dfc29cfdddca1"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:d2efee35b4b0a347e0d99d28e884dfd82797852d62fcd7ebdeee26f3ceb72cf3"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:6dcfcffe73710be01d90cae08c3e548d90932d37b39ef83969ae135d36ef3956"}, - {file = "wrapt-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:eb6e651000a19c96f452c85132811d25e9264d836951022d6e81df2fff38337d"}, - {file = "wrapt-1.16.0-cp311-cp311-win32.whl", hash = "sha256:66027d667efe95cc4fa945af59f92c5a02c6f5bb6012bff9e60542c74c75c362"}, - {file = "wrapt-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:aefbc4cb0a54f91af643660a0a150ce2c090d3652cf4052a5397fb2de549cd89"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5eb404d89131ec9b4f748fa5cfb5346802e5ee8836f57d516576e61f304f3b7b"}, - {file = "wrapt-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:9090c9e676d5236a6948330e83cb89969f433b1943a558968f659ead07cb3b36"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94265b00870aa407bd0cbcfd536f17ecde43b94fb8d228560a1e9d3041462d73"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2058f813d4f2b5e3a9eb2eb3faf8f1d99b81c3e51aeda4b168406443e8ba809"}, - {file = "wrapt-1.16.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98b5e1f498a8ca1858a1cdbffb023bfd954da4e3fa2c0cb5853d40014557248b"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:14d7dc606219cdd7405133c713f2c218d4252f2a469003f8c46bb92d5d095d81"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:49aac49dc4782cb04f58986e81ea0b4768e4ff197b57324dcbd7699c5dfb40b9"}, - {file = "wrapt-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:418abb18146475c310d7a6dc71143d6f7adec5b004ac9ce08dc7a34e2babdc5c"}, - {file = "wrapt-1.16.0-cp312-cp312-win32.whl", hash = "sha256:685f568fa5e627e93f3b52fda002c7ed2fa1800b50ce51f6ed1d572d8ab3e7fc"}, - {file = "wrapt-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:dcdba5c86e368442528f7060039eda390cc4091bfd1dca41e8046af7c910dda8"}, - {file = "wrapt-1.16.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d462f28826f4657968ae51d2181a074dfe03c200d6131690b7d65d55b0f360f8"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a33a747400b94b6d6b8a165e4480264a64a78c8a4c734b62136062e9a248dd39"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3646eefa23daeba62643a58aac816945cadc0afaf21800a1421eeba5f6cfb9c"}, - {file = "wrapt-1.16.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ebf019be5c09d400cf7b024aa52b1f3aeebeff51550d007e92c3c1c4afc2a40"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:0d2691979e93d06a95a26257adb7bfd0c93818e89b1406f5a28f36e0d8c1e1fc"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:1acd723ee2a8826f3d53910255643e33673e1d11db84ce5880675954183ec47e"}, - {file = "wrapt-1.16.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc57efac2da352a51cc4658878a68d2b1b67dbe9d33c36cb826ca449d80a8465"}, - {file = "wrapt-1.16.0-cp36-cp36m-win32.whl", hash = "sha256:da4813f751142436b075ed7aa012a8778aa43a99f7b36afe9b742d3ed8bdc95e"}, - {file = "wrapt-1.16.0-cp36-cp36m-win_amd64.whl", hash = "sha256:6f6eac2360f2d543cc875a0e5efd413b6cbd483cb3ad7ebf888884a6e0d2e966"}, - {file = "wrapt-1.16.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a0ea261ce52b5952bf669684a251a66df239ec6d441ccb59ec7afa882265d593"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bd2d7ff69a2cac767fbf7a2b206add2e9a210e57947dd7ce03e25d03d2de292"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9159485323798c8dc530a224bd3ffcf76659319ccc7bbd52e01e73bd0241a0c5"}, - {file = "wrapt-1.16.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a86373cf37cd7764f2201b76496aba58a52e76dedfaa698ef9e9688bfd9e41cf"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:73870c364c11f03ed072dda68ff7aea6d2a3a5c3fe250d917a429c7432e15228"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:b935ae30c6e7400022b50f8d359c03ed233d45b725cfdd299462f41ee5ffba6f"}, - {file = "wrapt-1.16.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:db98ad84a55eb09b3c32a96c576476777e87c520a34e2519d3e59c44710c002c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win32.whl", hash = "sha256:9153ed35fc5e4fa3b2fe97bddaa7cbec0ed22412b85bcdaf54aeba92ea37428c"}, - {file = "wrapt-1.16.0-cp37-cp37m-win_amd64.whl", hash = "sha256:66dfbaa7cfa3eb707bbfcd46dab2bc6207b005cbc9caa2199bcbc81d95071a00"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1dd50a2696ff89f57bd8847647a1c363b687d3d796dc30d4dd4a9d1689a706f0"}, - {file = "wrapt-1.16.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44a2754372e32ab315734c6c73b24351d06e77ffff6ae27d2ecf14cf3d229202"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e9723528b9f787dc59168369e42ae1c3b0d3fadb2f1a71de14531d321ee05b0"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbed418ba5c3dce92619656802cc5355cb679e58d0d89b50f116e4a9d5a9603e"}, - {file = "wrapt-1.16.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:941988b89b4fd6b41c3f0bfb20e92bd23746579736b7343283297c4c8cbae68f"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6a42cd0cfa8ffc1915aef79cb4284f6383d8a3e9dcca70c445dcfdd639d51267"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ca9b6085e4f866bd584fb135a041bfc32cab916e69f714a7d1d397f8c4891ca"}, - {file = "wrapt-1.16.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5e49454f19ef621089e204f862388d29e6e8d8b162efce05208913dde5b9ad6"}, - {file = "wrapt-1.16.0-cp38-cp38-win32.whl", hash = "sha256:c31f72b1b6624c9d863fc095da460802f43a7c6868c5dda140f51da24fd47d7b"}, - {file = "wrapt-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:490b0ee15c1a55be9c1bd8609b8cecd60e325f0575fc98f50058eae366e01f41"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9b201ae332c3637a42f02d1045e1d0cccfdc41f1f2f801dafbaa7e9b4797bfc2"}, - {file = "wrapt-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2076fad65c6736184e77d7d4729b63a6d1ae0b70da4868adeec40989858eb3fb"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5cd603b575ebceca7da5a3a251e69561bec509e0b46e4993e1cac402b7247b8"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b47cfad9e9bbbed2339081f4e346c93ecd7ab504299403320bf85f7f85c7d46c"}, - {file = "wrapt-1.16.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f8212564d49c50eb4565e502814f694e240c55551a5f1bc841d4fcaabb0a9b8a"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5f15814a33e42b04e3de432e573aa557f9f0f56458745c2074952f564c50e664"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db2e408d983b0e61e238cf579c09ef7020560441906ca990fe8412153e3b291f"}, - {file = "wrapt-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:edfad1d29c73f9b863ebe7082ae9321374ccb10879eeabc84ba3b69f2579d537"}, - {file = "wrapt-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed867c42c268f876097248e05b6117a65bcd1e63b779e916fe2e33cd6fd0d3c3"}, - {file = "wrapt-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:eb1b046be06b0fce7249f1d025cd359b4b80fc1c3e24ad9eca33e0dcdb2e4a35"}, - {file = "wrapt-1.16.0-py3-none-any.whl", hash = "sha256:6906c4100a8fcbf2fa735f6059214bb13b97f75b1a61777fcf6432121ef12ef1"}, - {file = "wrapt-1.16.0.tar.gz", hash = "sha256:5f370f952971e7d17c7d1ead40e49f32345a7f7a5373571ef44d800d06b1899d"}, -] - [[package]] name = "zipp" version = "3.18.1" @@ -1171,4 +948,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "00da1bbb044e832cc64594caa60a48c4340b65c35ad92c3094637b7a61024841" +content-hash = "81889e1ba9d27beae520791b9ecb3ec3887790d7df433cfe18068d842379ecea" diff --git a/pyproject.toml b/pyproject.toml index def86519..c4c074c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,10 +38,23 @@ importlib-resources = "^6.1.1" devchat = "devchat._cli.main:main" [tool.poetry.group.dev.dependencies] -pylint = "^2.17.4" pytest = "^7.4.0" -gitpython = "^3.1.32" +ruff = "^0.4.4" [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.ruff] +target-version = "py38" +line-length = 100 + +[tool.ruff.lint] +select = [ + "E", # Error + "W", # Warning + "F", # pyflakes + "I", # isort +] +fixable = ["ALL"] + diff --git a/scripts/purge_topics.py b/scripts/purge_topics.py index c32f95c0..5f4e7bef 100644 --- a/scripts/purge_topics.py +++ b/scripts/purge_topics.py @@ -1,12 +1,13 @@ import sys + from tinydb import TinyDB def remove_topic_table(file_path: str): try: db = TinyDB(file_path) - if 'topics' in db.tables(): - db.drop_table('topics') + if "topics" in db.tables(): + db.drop_table("topics") print("The 'topics' table has been removed.") else: print("The file does not contain a 'topics' table.") diff --git a/tests/conftest.py b/tests/conftest.py index 78949274..2a8fb25e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,11 +2,12 @@ import shutil import tempfile from pathlib import Path + import pytest from git import Repo -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def git_repo(request): # Create a temporary directory repo_dir = tempfile.mkdtemp() @@ -27,19 +28,19 @@ def cleanup(): return repo_dir -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def mock_home_dir(tmp_path, request): - home_dir = Path(tmp_path / 'home') + home_dir = Path(tmp_path / "home") home_dir.mkdir() - original_home = os.environ.get('HOME') - os.environ['HOME'] = str(home_dir) + original_home = os.environ.get("HOME") + os.environ["HOME"] = str(home_dir) def cleanup(): if original_home is not None: - os.environ['HOME'] = original_home + os.environ["HOME"] = original_home else: - del os.environ['HOME'] + del os.environ["HOME"] shutil.rmtree(home_dir) request.addfinalizer(cleanup) diff --git a/tests/test_cli_log.py b/tests/test_cli_log.py index bb34fe6f..932a9d4a 100755 --- a/tests/test_cli_log.py +++ b/tests/test_cli_log.py @@ -1,20 +1,22 @@ import json + from click.testing import CliRunner -from devchat.utils import get_prompt_hash, get_content + from devchat._cli.main import main +from devchat.utils import get_content, get_prompt_hash runner = CliRunner() -def test_log_no_args(git_repo): # pylint: disable=W0613 - result = runner.invoke(main, ['log']) +def test_log_no_args(git_repo): + result = runner.invoke(main, ["log"]) assert result.exit_code == 0 logs = json.loads(result.output) assert isinstance(logs, list) -def test_log_with_skip_and_max_count(git_repo): # pylint: disable=W0613 - result = runner.invoke(main, ['log', '--skip', '1', '--max-count', '2']) +def test_log_with_skip_and_max_count(git_repo): + result = runner.invoke(main, ["log", "--skip", "1", "--max-count", "2"]) assert result.exit_code == 0 logs = json.loads(result.output) assert isinstance(logs, list) @@ -37,58 +39,54 @@ def _within_range(num1: int, num2: int) -> bool: return False -def test_tokens_with_log(git_repo): # pylint: disable=W0613 - request1 = "Translate the following paragraph to Chinese. " \ - "Reply only such translation without any other words: " \ - "THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, " \ - "INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " \ - "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT." \ - "IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " \ - "OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, " \ +def test_tokens_with_log(git_repo): + request1 = ( + "Translate the following paragraph to Chinese. " + "Reply only such translation without any other words: " + "THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, " + "INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, " + "FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT." + "IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR " + "OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, " "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE." - request2 = "Write a Hello World program in C++. Put the code in a block. " \ + ) + request2 = ( + "Write a Hello World program in C++. Put the code in a block. " "Do not reply any other words." + ) # Group 1 config_str = '--config={ "stream": false, "temperature": 0 }' - result = runner.invoke( - main, - ['prompt', '--model=gpt-3.5-turbo', config_str, request1] - ) + result = runner.invoke(main, ["prompt", "--model=gpt-3.5-turbo", config_str, request1]) assert result.exit_code == 0 parent1 = get_prompt_hash(result.output) config_str = '--config={ "stream": true, "temperature": 0 }' - result = runner.invoke( - main, - ['prompt', '--model=gpt-3.5-turbo', config_str, request1] - ) + result = runner.invoke(main, ["prompt", "--model=gpt-3.5-turbo", config_str, request1]) assert result.exit_code == 0 parent2 = get_prompt_hash(result.output) - result = runner.invoke(main, ['log', '-n', '2']) + result = runner.invoke(main, ["log", "-n", "2"]) logs = json.loads(result.output) assert logs[0]["hash"] == parent2 # Group 2 result = runner.invoke( - main, - ['prompt', '--model=gpt-3.5-turbo', config_str, '-p', parent1, request2] + main, ["prompt", "--model=gpt-3.5-turbo", config_str, "-p", parent1, request2] ) assert result.exit_code == 0 result = runner.invoke( - main, - ['prompt', '--model=gpt-3.5-turbo', config_str, '-p', parent2, request2] + main, ["prompt", "--model=gpt-3.5-turbo", config_str, "-p", parent2, request2] ) assert result.exit_code == 0 - result = runner.invoke(main, ['log', '-n', '2']) + result = runner.invoke(main, ["log", "-n", "2"]) logs = json.loads(result.output) assert len(logs) > 0 -def test_log_insert(git_repo): # pylint: disable=W0613 +def test_log_insert(git_repo): chat1 = """{ "model": "gpt-3.5-turbo", "messages": [ @@ -105,10 +103,7 @@ def test_log_insert(git_repo): # pylint: disable=W0613 "request_tokens": 100, "response_tokens": 100 }""" - result = runner.invoke( - main, - ['log', '--insert', chat1] - ) + result = runner.invoke(main, ["log", "--insert", chat1]) assert result.exit_code == 0 prompt1 = json.loads(result.output)[0] @@ -128,10 +123,7 @@ def test_log_insert(git_repo): # pylint: disable=W0613 "request_tokens": 200, "response_tokens": 200 }""" - result = runner.invoke( - main, - ['log', '--insert', chat2] - ) + result = runner.invoke(main, ["log", "--insert", chat2]) assert result.exit_code == 0 prompt2 = json.loads(result.output)[0] @@ -152,36 +144,32 @@ def test_log_insert(git_repo): # pylint: disable=W0613 "request_tokens": 300, "response_tokens": 300 }}""" - result = runner.invoke( - main, - ['log', '--insert', chat3] - ) + result = runner.invoke(main, ["log", "--insert", chat3]) assert result.exit_code == 0 prompt3 = json.loads(result.output)[0] - assert prompt3['parent'] == prompt1['hash'] + assert prompt3["parent"] == prompt1["hash"] - result = runner.invoke(main, ['log', '-n', '3', '-t', prompt1['hash']]) + result = runner.invoke(main, ["log", "-n", "3", "-t", prompt1["hash"]]) logs = json.loads(result.output) - assert logs[0]['hash'] == prompt3['hash'] - assert logs[1]['hash'] == prompt1['hash'] + assert logs[0]["hash"] == prompt3["hash"] + assert logs[1]["hash"] == prompt1["hash"] - result = runner.invoke(main, ['topic', '--list']) + result = runner.invoke(main, ["topic", "--list"]) topics = json.loads(result.output) - prompt_hashes = [prompt1['hash'], prompt2['hash']] - topics = [topic for topic in topics if topic['root_prompt']['hash'] in prompt_hashes] - assert topics[0]['root_prompt']['hash'] == prompt1['hash'] - assert topics[1]['root_prompt']['hash'] == prompt2['hash'] + prompt_hashes = [prompt1["hash"], prompt2["hash"]] + topics = [topic for topic in topics if topic["root_prompt"]["hash"] in prompt_hashes] + assert topics[0]["root_prompt"]["hash"] == prompt1["hash"] + assert topics[1]["root_prompt"]["hash"] == prompt2["hash"] result = runner.invoke( - main, - ['prompt', '-p', prompt2['hash'], 'Again, just reply the topic number.'] + main, ["prompt", "-p", prompt2["hash"], "Again, just reply the topic number."] ) assert result.exit_code == 0 content = get_content(result.output) assert content.strip() == "Topic 2" or content.strip() == "2" - result = runner.invoke(main, ['log', '-t', prompt2['hash'], '-n', '100']) + result = runner.invoke(main, ["log", "-t", prompt2["hash"], "-n", "100"]) assert result.exit_code == 0 logs = json.loads(result.output) assert len(logs) == 2 - assert logs[0]['responses'][0] == "Topic 2" or logs[0]['responses'][0] == "2" + assert logs[0]["responses"][0] == "Topic 2" or logs[0]["responses"][0] == "2" diff --git a/tests/test_cli_prompt.py b/tests/test_cli_prompt.py index 2759c60b..ee2ae61e 100644 --- a/tests/test_cli_prompt.py +++ b/tests/test_cli_prompt.py @@ -1,23 +1,24 @@ -import os import json +import os + import pytest from click.testing import CliRunner -from devchat.config import ConfigManager, GeneralModelConfig + from devchat._cli.main import main -from devchat.utils import openai_response_tokens -from devchat.utils import check_format, get_content, get_prompt_hash +from devchat.config import ConfigManager, GeneralModelConfig +from devchat.utils import check_format, get_content, get_prompt_hash, openai_response_tokens runner = CliRunner() -# def test_prompt_no_args(git_repo): # pylint: disable=W0613 +# def test_prompt_no_args(git_repo): # result = runner.invoke(main, ['prompt']) # assert result.exit_code == 0 -def test_prompt_with_content(git_repo): # pylint: disable=W0613 +def test_prompt_with_content(git_repo): content = "What is the capital of France?" - result = runner.invoke(main, ['prompt', content]) + result = runner.invoke(main, ["prompt", content]) print(result.output) assert result.exit_code == 0 assert check_format(result.output) @@ -41,11 +42,11 @@ def test_prompt_with_temp_config_file(mock_home_dir): os.makedirs(chat_dir) config_path = os.path.join(chat_dir, "config.yml") - with open(config_path, "w", encoding='utf-8') as config_file: + with open(config_path, "w", encoding="utf-8") as config_file: config_file.write(config_data) content = "What is the capital of Spain?" - result = runner.invoke(main, ['prompt', content]) + result = runner.invoke(main, ["prompt", content]) print(result.output) assert result.exit_code == 0 assert check_format(result.output) @@ -54,13 +55,15 @@ def test_prompt_with_temp_config_file(mock_home_dir): @pytest.fixture(name="temp_files") def fixture_temp_files(tmpdir): - instruct0 = tmpdir.join('instruct0.txt') + instruct0 = tmpdir.join("instruct0.txt") instruct0.write("Summarize the following user message.\n") - instruct1 = tmpdir.join('instruct1.txt') + instruct1 = tmpdir.join("instruct1.txt") instruct1.write("The summary must be lowercased under 5 characters without any punctuation.\n") - instruct2 = tmpdir.join('instruct2.txt') - instruct2.write("The summary must be lowercased under 10 characters without any punctuation." - "Include the information in to create the summary.\n") + instruct2 = tmpdir.join("instruct2.txt") + instruct2.write( + "The summary must be lowercased under 10 characters without any punctuation." + "Include the information in to create the summary.\n" + ) context = tmpdir.join("context.txt") context.write("It is summer.") return str(instruct0), str(instruct1), str(instruct2), str(context) @@ -68,7 +71,7 @@ def fixture_temp_files(tmpdir): @pytest.fixture(name="functions_file") def fixture_functions_file(tmpdir): - functions_file = tmpdir.join('functions.json') + functions_file = tmpdir.join("functions.json") functions_file.write(""" [ { @@ -94,60 +97,100 @@ def fixture_functions_file(tmpdir): return str(functions_file) -def test_prompt_with_instruct(git_repo, temp_files): # pylint: disable=W0613 - result = runner.invoke(main, ['prompt', '-m', 'gpt-4', - '-i', temp_files[0], '-i', temp_files[1], - "It is really scorching."]) +def test_prompt_with_instruct(git_repo, temp_files): + result = runner.invoke( + main, + [ + "prompt", + "-m", + "gpt-4", + "-i", + temp_files[0], + "-i", + temp_files[1], + "It is really scorching.", + ], + ) assert result.exit_code == 0 assert get_content(result.output).find("hot\n") >= 0 -def test_prompt_with_instruct_and_context(git_repo, temp_files): # pylint: disable=W0613 - result = runner.invoke(main, ['prompt', '-m', 'gpt-4', - '-i', temp_files[0], '-i', temp_files[2], - '--context', temp_files[3], - "It is really scorching."]) +def test_prompt_with_instruct_and_context(git_repo, temp_files): + result = runner.invoke( + main, + [ + "prompt", + "-m", + "gpt-4", + "-i", + temp_files[0], + "-i", + temp_files[2], + "--context", + temp_files[3], + "It is really scorching.", + ], + ) assert result.exit_code == 0 assert get_content(result.output).find("hot summer\n") >= 0 -def test_prompt_with_functions(git_repo, functions_file): # pylint: disable=W0613 +def test_prompt_with_functions(git_repo, functions_file): # call with -f option - result = runner.invoke(main, ['prompt', '-m', 'gpt-3.5-turbo', '-f', functions_file, - "What is the weather like in Boston?"]) + result = runner.invoke( + main, + [ + "prompt", + "-m", + "gpt-3.5-turbo", + "-f", + functions_file, + "What is the weather like in Boston?", + ], + ) if result.exit_code: print(result.output) assert result.exit_code == 0 content = get_content(result.output) - assert 'finish_reason: function_call' in content - assert '```command' in content + assert "finish_reason: function_call" in content + assert "```command" in content assert '"name": "get_current_weather"' in content # compare with no -f options - result = runner.invoke(main, ['prompt', '-m', 'gpt-3.5-turbo', - 'What is the weather like in Boston?']) + result = runner.invoke( + main, ["prompt", "-m", "gpt-3.5-turbo", "What is the weather like in Boston?"] + ) content = get_content(result.output) assert result.exit_code == 0 - assert 'finish_reason: stop' not in content - assert 'command' not in content + assert "finish_reason: stop" not in content + assert "command" not in content -def test_prompt_log_with_functions(git_repo, functions_file): # pylint: disable=W0613 +def test_prompt_log_with_functions(git_repo, functions_file): # call with -f option - result = runner.invoke(main, ['prompt', '-m', 'gpt-3.5-turbo', '-f', functions_file, - 'What is the weather like in Boston?']) + result = runner.invoke( + main, + [ + "prompt", + "-m", + "gpt-3.5-turbo", + "-f", + functions_file, + "What is the weather like in Boston?", + ], + ) if result.exit_code: print(result.output) assert result.exit_code == 0 prompt_hash = get_prompt_hash(result.output) - result = runner.invoke(main, ['log', '-t', prompt_hash]) + result = runner.invoke(main, ["log", "-t", prompt_hash]) result_json = json.loads(result.output) assert result.exit_code == 0 - assert result_json[0]['request'] == 'What is the weather like in Boston?' - assert '```command' in result_json[0]['responses'][0] - assert 'get_current_weather' in result_json[0]['responses'][0] + assert result_json[0]["request"] == "What is the weather like in Boston?" + assert "```command" in result_json[0]["responses"][0] + assert "get_current_weather" in result_json[0]["responses"][0] def test_prompt_log_compatibility(): @@ -166,44 +209,53 @@ def test_prompt_log_compatibility(): # test prompt with function replay -def test_prompt_with_function_replay(git_repo, functions_file): # pylint: disable=W0613 - result = runner.invoke(main, ['prompt', '-m', 'gpt-3.5-turbo', - '-f', functions_file, - '-n', 'get_current_weather', - '{"temperature": "22", "unit": "celsius", "weather": "Sunny"}']) +def test_prompt_with_function_replay(git_repo, functions_file): + result = runner.invoke( + main, + [ + "prompt", + "-m", + "gpt-3.5-turbo", + "-f", + functions_file, + "-n", + "get_current_weather", + '{"temperature": "22", "unit": "celsius", "weather": "Sunny"}', + ], + ) content = get_content(result.output) assert result.exit_code == 0 - assert '22' in content - assert 'sunny' in content or 'Sunny' in content + assert "22" in content + assert "sunny" in content or "Sunny" in content prompt_hash = get_prompt_hash(result.output) - result = runner.invoke(main, ['prompt', '-m', 'gpt-3.5-turbo', - '-p', prompt_hash, - 'what is the function tool name?']) + result = runner.invoke( + main, + ["prompt", "-m", "gpt-3.5-turbo", "-p", prompt_hash, "what is the function tool name?"], + ) content = get_content(result.output) assert result.exit_code == 0 - assert 'get_current_weather' in content \ - or 'GetCurrentWeather' in content \ - or 'getCurrentWeather' in content + assert ( + "get_current_weather" in content + or "GetCurrentWeather" in content + or "getCurrentWeather" in content + ) -def test_prompt_without_repo(mock_home_dir): # pylint: disable=W0613 +def test_prompt_without_repo(mock_home_dir): content = "What is the capital of France?" - result = runner.invoke(main, ['prompt', content]) + result = runner.invoke(main, ["prompt", content]) assert result.exit_code == 0 assert check_format(result.output) assert "Paris" in result.output -def test_prompt_tokens_exceed_config(mock_home_dir): # pylint: disable=W0613 +def test_prompt_tokens_exceed_config(mock_home_dir): model = "gpt-3.5-turbo" max_input_tokens = 4000 - config = GeneralModelConfig( - max_input_tokens=200, - temperature=0 - ) + config = GeneralModelConfig(max_input_tokens=200, temperature=0) chat_dir = os.path.join(mock_home_dir, ".chat") os.makedirs(chat_dir) @@ -215,19 +267,16 @@ def test_prompt_tokens_exceed_config(mock_home_dir): # pylint: disable=W0613 content = "" while openai_response_tokens({"content": content}, model) < max_input_tokens: content += "This is a test. Ignore what I say.\n" - result = runner.invoke(main, ['prompt', content]) + result = runner.invoke(main, ["prompt", content]) print(result.output) assert result.exit_code != 0 assert "beyond limit" in result.output -def test_file_tokens_exceed_config(mock_home_dir, tmpdir): # pylint: disable=W0613 +def test_file_tokens_exceed_config(mock_home_dir, tmpdir): model = "gpt-3.5-turbo" max_input_tokens = 4000 - config = GeneralModelConfig( - max_input_tokens=200, - temperature=0 - ) + config = GeneralModelConfig(max_input_tokens=200, temperature=0) chat_dir = os.path.join(mock_home_dir, ".chat") os.makedirs(chat_dir) @@ -243,6 +292,6 @@ def test_file_tokens_exceed_config(mock_home_dir, tmpdir): # pylint: disable=W0 content_file.write(content) input_str = "This is a test. Ignore what I say." - result = runner.invoke(main, ['prompt', '-c', str(content_file), input_str]) + result = runner.invoke(main, ["prompt", "-c", str(content_file), input_str]) assert result.exit_code != 0 assert "beyond limit" in result.output diff --git a/tests/test_cli_topic.py b/tests/test_cli_topic.py index 595d6237..0f47a88a 100644 --- a/tests/test_cli_topic.py +++ b/tests/test_cli_topic.py @@ -1,37 +1,33 @@ import json import sys import time + from click.testing import CliRunner -from devchat.utils import get_prompt_hash + from devchat._cli.main import main +from devchat.utils import get_prompt_hash runner = CliRunner() -def test_topic_list(git_repo): # pylint: disable=W0613 +def test_topic_list(git_repo): request = "Complete the sequence 1, 1, 3, 5, 9, ( ). Reply the number only." - sys.argv = ['prompt', '--model=gpt-3.5-turbo', request] - result = runner.invoke( - main, - ['prompt', '--model=gpt-3.5-turbo', request] - ) + sys.argv = ["prompt", "--model=gpt-3.5-turbo", request] + result = runner.invoke(main, ["prompt", "--model=gpt-3.5-turbo", request]) assert result.exit_code == 0 topic1 = get_prompt_hash(result.output) time.sleep(3) - result = runner.invoke( - main, - ['prompt', '--model=gpt-4', request] - ) + result = runner.invoke(main, ["prompt", "--model=gpt-4", request]) assert result.exit_code == 0 topic2 = get_prompt_hash(result.output) - result = runner.invoke(main, ['topic', '--list']) + result = runner.invoke(main, ["topic", "--list"]) assert result.exit_code == 0 topics = json.loads(result.output) assert len(topics) >= 2 - assert topics[0]['root_prompt']['responses'][0] == "15" - assert topics[1]['root_prompt']['responses'][0] == "17" - assert topics[0]['root_prompt']['hash'] == topic2 - assert topics[1]['root_prompt']['hash'] == topic1 + assert topics[0]["root_prompt"]["responses"][0] == "15" + assert topics[1]["root_prompt"]["responses"][0] == "17" + assert topics[0]["root_prompt"]["hash"] == topic2 + assert topics[1]["root_prompt"]["hash"] == topic1 diff --git a/tests/test_command_parser.py b/tests/test_command_parser.py index 740d4c3f..b79dd2d5 100644 --- a/tests/test_command_parser.py +++ b/tests/test_command_parser.py @@ -1,13 +1,14 @@ import os import tempfile + import pytest -from devchat.engine import Command, parse_command -from devchat.engine import CommandParser, Namespace + +from devchat.engine import Command, CommandParser, Namespace, parse_command def test_parse_command(): # Test with a valid configuration file with most fields filled - with tempfile.NamedTemporaryFile('w', delete=False) as config_file: + with tempfile.NamedTemporaryFile("w", delete=False) as config_file: config_file.write(""" description: Get the current weather in a given location parameters: @@ -26,13 +27,13 @@ def test_parse_command(): command = parse_command(config_file.name) assert isinstance(command, Command) command = command.dict() - assert command['description'] == 'Get the current weather in a given location' - assert 'location' in command['parameters'] - assert command['parameters']['unit']['default'] == 'celsius' - assert command['steps'][0]['run'] == './get_weather --location=$location --unit=$unit' + assert command["description"] == "Get the current weather in a given location" + assert "location" in command["parameters"] + assert command["parameters"]["unit"]["default"] == "celsius" + assert command["steps"][0]["run"] == "./get_weather --location=$location --unit=$unit" # Test with a valid configuration file with missing optional fields - with tempfile.NamedTemporaryFile('w', delete=False) as config_file: + with tempfile.NamedTemporaryFile("w", delete=False) as config_file: config_file.write(""" description: Prompt for /code parameters: @@ -43,7 +44,7 @@ def test_parse_command(): assert command.steps is None # Test with an invalid configuration file - with tempfile.NamedTemporaryFile('w', delete=False) as config_file: + with tempfile.NamedTemporaryFile("w", delete=False) as config_file: config_file.write(""" description: parameters: @@ -56,7 +57,7 @@ def test_parse_command(): # Test with a non-existent file with pytest.raises(FileNotFoundError): - parse_command('path/to/non_existent_file.yml') + parse_command("path/to/non_existent_file.yml") def test_command_parser(tmp_path): @@ -65,9 +66,9 @@ def test_command_parser(tmp_path): command_parser = CommandParser(namespace) # Test with a valid configuration file with most fields filled - os.makedirs(os.path.join(tmp_path, 'usr', 'a', 'b', 'c'), exist_ok=True) - command_file_path = os.path.join(tmp_path, 'usr', 'a', 'b', 'c', 'command.yml') - with open(command_file_path, 'w', encoding='utf-8') as file: + os.makedirs(os.path.join(tmp_path, "usr", "a", "b", "c"), exist_ok=True) + command_file_path = os.path.join(tmp_path, "usr", "a", "b", "c", "command.yml") + with open(command_file_path, "w", encoding="utf-8") as file: file.write(""" description: Get the current weather in a given location parameters: @@ -82,31 +83,31 @@ def test_command_parser(tmp_path): - run: ./get_weather --location=$location --unit=$unit """) - command = command_parser.parse('a.b.c') + command = command_parser.parse("a.b.c") command = command.dict() - assert command['description'] == 'Get the current weather in a given location' - assert 'location' in command['parameters'] - assert command['parameters']['unit']['default'] == 'celsius' - assert command['steps'][0]['run'] == './get_weather --location=$location --unit=$unit' + assert command["description"] == "Get the current weather in a given location" + assert "location" in command["parameters"] + assert command["parameters"]["unit"]["default"] == "celsius" + assert command["steps"][0]["run"] == "./get_weather --location=$location --unit=$unit" # Test with a valid configuration file with missing optional fields - os.makedirs(os.path.join(tmp_path, 'usr', 'd', 'e', 'f'), exist_ok=True) - command_file_path = os.path.join(tmp_path, 'usr', 'd', 'e', 'f', 'command.yml') - with open(command_file_path, 'w', encoding='utf-8') as file: + os.makedirs(os.path.join(tmp_path, "usr", "d", "e", "f"), exist_ok=True) + command_file_path = os.path.join(tmp_path, "usr", "d", "e", "f", "command.yml") + with open(command_file_path, "w", encoding="utf-8") as file: file.write(""" description: Prompt for /code parameters: """) - command = command_parser.parse('d.e.f') + command = command_parser.parse("d.e.f") command = command.dict() - assert command['description'] == 'Prompt for /code' - assert command['parameters'] is None - assert command['steps'] is None + assert command["description"] == "Prompt for /code" + assert command["parameters"] is None + assert command["steps"] is None # Test with an invalid configuration file - os.makedirs(os.path.join(tmp_path, 'usr', 'g', 'h', 'i'), exist_ok=True) - command_file_path = os.path.join(tmp_path, 'usr', 'g', 'h', 'i', 'command.yml') - with open(command_file_path, 'w', encoding='utf-8') as file: + os.makedirs(os.path.join(tmp_path, "usr", "g", "h", "i"), exist_ok=True) + command_file_path = os.path.join(tmp_path, "usr", "g", "h", "i", "command.yml") + with open(command_file_path, "w", encoding="utf-8") as file: file.write(""" description: parameters: @@ -114,8 +115,8 @@ def test_command_parser(tmp_path): type: string """) with pytest.raises(Exception): - command_parser.parse('g.h.i') + command_parser.parse("g.h.i") # Test with a non-existent command - command = command_parser.parse('j.k.l') + command = command_parser.parse("j.k.l") assert command is None diff --git a/tests/test_config.py b/tests/test_config.py index 1ff7dd4c..997e5a63 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,13 +1,15 @@ import os + from click.testing import CliRunner -from devchat.config import ConfigManager, GeneralModelConfig, ChatConfig + +from devchat.config import ChatConfig, ConfigManager, GeneralModelConfig runner = CliRunner() def test_create_sample_config(tmp_path): ConfigManager(tmp_path) - assert os.path.exists(os.path.join(tmp_path, 'config.yml')) + assert os.path.exists(os.path.join(tmp_path, "config.yml")) def test_load_and_validate_config(tmp_path): @@ -17,19 +19,16 @@ def test_load_and_validate_config(tmp_path): def test_get_model_config(tmp_path): config_manager = ConfigManager(tmp_path) - _, config = config_manager.model_config('gpt-4') + _, config = config_manager.model_config("gpt-4") assert config.max_input_tokens == 6000 assert config.temperature == 0 assert config.stream is True def test_update_model_config(tmp_path): - model = 'gpt-4' + model = "gpt-4" config_manager = ConfigManager(tmp_path) - config = GeneralModelConfig( - max_input_tokens=7000, - temperature=0.5 - ) + config = GeneralModelConfig(max_input_tokens=7000, temperature=0.5) updated_model_config = config_manager.update_model_config(model, config) assert updated_model_config == config_manager.model_config(model)[1] assert updated_model_config.max_input_tokens == 7000 diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 364e2a63..b18c22f8 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -1,33 +1,35 @@ import os + import pytest + from devchat.engine import Namespace def test_is_valid_name(): # Test valid names - assert Namespace.is_valid_name('') is True - assert Namespace.is_valid_name('a') is True - assert Namespace.is_valid_name('A.b') is True - assert Namespace.is_valid_name('a.2.c') is True - assert Namespace.is_valid_name('a_b') is True - assert Namespace.is_valid_name('a-b') is True - assert Namespace.is_valid_name('a_3.4-d') is True + assert Namespace.is_valid_name("") is True + assert Namespace.is_valid_name("a") is True + assert Namespace.is_valid_name("A.b") is True + assert Namespace.is_valid_name("a.2.c") is True + assert Namespace.is_valid_name("a_b") is True + assert Namespace.is_valid_name("a-b") is True + assert Namespace.is_valid_name("a_3.4-d") is True # Test invalid names - assert Namespace.is_valid_name('.') is False - assert Namespace.is_valid_name('..') is False - assert Namespace.is_valid_name('a..b') is False - assert Namespace.is_valid_name('.a') is False - assert Namespace.is_valid_name('3.') is False - assert Namespace.is_valid_name('a/.b') is False - assert Namespace.is_valid_name('a\\b') is False - assert Namespace.is_valid_name('a*b') is False - assert Namespace.is_valid_name('a?1') is False - assert Namespace.is_valid_name('a:b') is False - assert Namespace.is_valid_name('a|b') is False + assert Namespace.is_valid_name(".") is False + assert Namespace.is_valid_name("..") is False + assert Namespace.is_valid_name("a..b") is False + assert Namespace.is_valid_name(".a") is False + assert Namespace.is_valid_name("3.") is False + assert Namespace.is_valid_name("a/.b") is False + assert Namespace.is_valid_name("a\\b") is False + assert Namespace.is_valid_name("a*b") is False + assert Namespace.is_valid_name("a?1") is False + assert Namespace.is_valid_name("a:b") is False + assert Namespace.is_valid_name("a|b") is False assert Namespace.is_valid_name('a"b') is False - assert Namespace.is_valid_name('2b') is False + assert Namespace.is_valid_name("2b") is False def test_get_file(tmp_path): @@ -36,35 +38,35 @@ def test_get_file(tmp_path): # Test case 1: a file that exists # Create a file in the 'usr' branch - os.makedirs(os.path.join(tmp_path, 'usr', 'a', 'b', 'c'), exist_ok=True) - file_path = os.path.join(tmp_path, 'usr', 'a', 'b', 'c', 'file1.txt') - with open(file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.get_file('a.b.c', 'file1.txt') == file_path + os.makedirs(os.path.join(tmp_path, "usr", "a", "b", "c"), exist_ok=True) + file_path = os.path.join(tmp_path, "usr", "a", "b", "c", "file1.txt") + with open(file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.get_file("a.b.c", "file1.txt") == file_path # Test case 2: a file that doesn't exist - assert namespace.get_file('d.e.f', 'file2.txt') is None + assert namespace.get_file("d.e.f", "file2.txt") is None # Test case 3: a file that exists in a later branch # Create a file in the 'sys' branch - os.makedirs(os.path.join(tmp_path, 'usr', 'g', 'h', 'i'), exist_ok=True) - os.makedirs(os.path.join(tmp_path, 'sys', 'g', 'h', 'i'), exist_ok=True) - file_path = os.path.join(tmp_path, 'sys', 'g', 'h', 'i', 'file3.txt') - with open(file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.get_file('g.h.i', 'file3.txt') == file_path + os.makedirs(os.path.join(tmp_path, "usr", "g", "h", "i"), exist_ok=True) + os.makedirs(os.path.join(tmp_path, "sys", "g", "h", "i"), exist_ok=True) + file_path = os.path.join(tmp_path, "sys", "g", "h", "i", "file3.txt") + with open(file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.get_file("g.h.i", "file3.txt") == file_path # Test case 4: a file in 'usr' overwrites the same in 'sys' # Create the same file in the 'usr' and 'sys' branches - os.makedirs(os.path.join(tmp_path, 'usr', 'j', 'k', 'l'), exist_ok=True) - usr_file_path = os.path.join(tmp_path, 'usr', 'j', 'k', 'l', 'file4.txt') - os.makedirs(os.path.join(tmp_path, 'sys', 'j', 'k', 'l'), exist_ok=True) - sys_file_path = os.path.join(tmp_path, 'sys', 'j', 'k', 'l', 'file4.txt') - with open(usr_file_path, 'w', encoding='utf-8') as file: - file.write('test') - with open(sys_file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.get_file('j.k.l', 'file4.txt') == usr_file_path + os.makedirs(os.path.join(tmp_path, "usr", "j", "k", "l"), exist_ok=True) + usr_file_path = os.path.join(tmp_path, "usr", "j", "k", "l", "file4.txt") + os.makedirs(os.path.join(tmp_path, "sys", "j", "k", "l"), exist_ok=True) + sys_file_path = os.path.join(tmp_path, "sys", "j", "k", "l", "file4.txt") + with open(usr_file_path, "w", encoding="utf-8") as file: + file.write("test") + with open(sys_file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.get_file("j.k.l", "file4.txt") == usr_file_path def test_list_files(tmp_path): @@ -73,65 +75,65 @@ def test_list_files(tmp_path): # Test case 1: a path that exists # Create a file in the 'usr' branch - os.makedirs(os.path.join(tmp_path, 'usr', 'a', 'b', 'c'), exist_ok=True) - file_path = os.path.join(tmp_path, 'usr', 'a', 'b', 'c', 'file1.txt') - with open(file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.list_files('a.b.c') == [file_path] + os.makedirs(os.path.join(tmp_path, "usr", "a", "b", "c"), exist_ok=True) + file_path = os.path.join(tmp_path, "usr", "a", "b", "c", "file1.txt") + with open(file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.list_files("a.b.c") == [file_path] # Test case 2: a path that doesn't exist with pytest.raises(ValueError): - namespace.list_files('d.e.f') + namespace.list_files("d.e.f") # Test case 3: a path exists but has no files - os.makedirs(os.path.join(tmp_path, 'org', 'd', 'e', 'f'), exist_ok=True) - assert not namespace.list_files('d.e.f') + os.makedirs(os.path.join(tmp_path, "org", "d", "e", "f"), exist_ok=True) + assert not namespace.list_files("d.e.f") # Test case 4: a path that exists in a later branch # Create a file in the 'sys' branch - os.makedirs(os.path.join(tmp_path, 'usr', 'g', 'h', 'i'), exist_ok=True) - os.makedirs(os.path.join(tmp_path, 'sys', 'g', 'h', 'i'), exist_ok=True) - file_path = os.path.join(tmp_path, 'sys', 'g', 'h', 'i', 'file2.txt') - with open(file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.list_files('g.h.i') == [file_path] + os.makedirs(os.path.join(tmp_path, "usr", "g", "h", "i"), exist_ok=True) + os.makedirs(os.path.join(tmp_path, "sys", "g", "h", "i"), exist_ok=True) + file_path = os.path.join(tmp_path, "sys", "g", "h", "i", "file2.txt") + with open(file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.list_files("g.h.i") == [file_path] # Test case 5: a path in 'usr' overwrites the same in 'sys' # Create the same file in the 'usr' and 'sys' branches - os.makedirs(os.path.join(tmp_path, 'usr', 'j', 'k', 'l'), exist_ok=True) - usr_file_path = os.path.join(tmp_path, 'usr', 'j', 'k', 'l', 'file3.txt') - os.makedirs(os.path.join(tmp_path, 'sys', 'j', 'k', 'l'), exist_ok=True) - sys_file_path = os.path.join(tmp_path, 'sys', 'j', 'k', 'l', 'file3.txt') - with open(usr_file_path, 'w', encoding='utf-8') as file: - file.write('test') - with open(sys_file_path, 'w', encoding='utf-8') as file: - file.write('test') - assert namespace.list_files('j.k.l') == [usr_file_path] + os.makedirs(os.path.join(tmp_path, "usr", "j", "k", "l"), exist_ok=True) + usr_file_path = os.path.join(tmp_path, "usr", "j", "k", "l", "file3.txt") + os.makedirs(os.path.join(tmp_path, "sys", "j", "k", "l"), exist_ok=True) + sys_file_path = os.path.join(tmp_path, "sys", "j", "k", "l", "file3.txt") + with open(usr_file_path, "w", encoding="utf-8") as file: + file.write("test") + with open(sys_file_path, "w", encoding="utf-8") as file: + file.write("test") + assert namespace.list_files("j.k.l") == [usr_file_path] def test_list_names(tmp_path): - os.makedirs(os.path.join(tmp_path, 'usr', 'a', 'b', 'c')) - os.makedirs(os.path.join(tmp_path, 'org', 'a', 'b', 'd')) - os.makedirs(os.path.join(tmp_path, 'sys', 'a', 'e')) + os.makedirs(os.path.join(tmp_path, "usr", "a", "b", "c")) + os.makedirs(os.path.join(tmp_path, "org", "a", "b", "d")) + os.makedirs(os.path.join(tmp_path, "sys", "a", "e")) namespace = Namespace(tmp_path) # Test listing child commands - commands = namespace.list_names('a') - assert commands == ['a.b', 'a.e'] + commands = namespace.list_names("a") + assert commands == ["a.b", "a.e"] # Test listing all descendant commands - commands = namespace.list_names('a', recursive=True) - assert commands == ['a.b', 'a.b.c', 'a.b.d', 'a.e'] + commands = namespace.list_names("a", recursive=True) + assert commands == ["a.b", "a.b.c", "a.b.d", "a.e"] # Test listing commands of an invalid name with pytest.raises(ValueError): - namespace.list_names('b') + namespace.list_names("b") # Test listing commands when there are no commands - commands = namespace.list_names('a.e') + commands = namespace.list_names("a.e") assert len(commands) == 0 # Test listing commands of the root commands = namespace.list_names() - assert commands == ['a'] + assert commands == ["a"] diff --git a/tests/test_openai_message.py b/tests/test_openai_message.py index f6eaaf27..d168078f 100644 --- a/tests/test_openai_message.py +++ b/tests/test_openai_message.py @@ -1,4 +1,5 @@ import pytest + from devchat.openai import OpenAIMessage @@ -45,10 +46,7 @@ def test_none_name(): def test_from_dict(): - message_data = { - "content": "Welcome to the chat.", - "role": "system" - } + message_data = {"content": "Welcome to the chat.", "role": "system"} message = OpenAIMessage.from_dict(message_data) assert message.role == "system" assert message.content == "Welcome to the chat." @@ -56,11 +54,7 @@ def test_from_dict(): def test_from_dict_with_name(): - message_data = { - "content": "Hello, Assistant!", - "role": "user", - "name": "JohnDoe" - } + message_data = {"content": "Hello, Assistant!", "role": "user", "name": "JohnDoe"} message = OpenAIMessage.from_dict(message_data) assert message.role == "user" assert message.content == "Hello, Assistant!" diff --git a/tests/test_openai_prompt.py b/tests/test_openai_prompt.py index d6492c7b..c3b2181e 100644 --- a/tests/test_openai_prompt.py +++ b/tests/test_openai_prompt.py @@ -1,10 +1,10 @@ -# pylint: disable=protected-access import json import os + import pytest + from devchat.message import Message -from devchat.openai import OpenAIMessage -from devchat.openai import OpenAIPrompt +from devchat.openai import OpenAIMessage, OpenAIPrompt from devchat.utils import get_user_info @@ -14,7 +14,7 @@ def test_prompt_init_and_set_response(): assert prompt.model == "gpt-3.5-turbo" prompt.set_request("Where was the 2020 World Series played?") - response_str = ''' + response_str = """ { "id": "chatcmpl-6p9XYPYSTTRi0xEviKjjilqrWU2Ve", "object": "chat.completion", @@ -32,7 +32,7 @@ def test_prompt_init_and_set_response(): } ] } - ''' + """ prompt.set_response(response_str) assert prompt.timestamp == 1677649420 @@ -43,17 +43,17 @@ def test_prompt_init_and_set_response(): assert prompt.responses[0].content == "The 2020 World Series was played in Arlington, Texas." -@pytest.fixture(scope="module", name='responses') +@pytest.fixture(scope="module", name="responses") def fixture_responses(): current_dir = os.path.dirname(__file__) folder_path = os.path.join(current_dir, "stream_responses") stream_responses = [] file_names = os.listdir(folder_path) - sorted_file_names = sorted(file_names, key=lambda x: int(x.split('.')[0][8:])) + sorted_file_names = sorted(file_names, key=lambda x: int(x.split(".")[0][8:])) for file_name in sorted_file_names: - with open(os.path.join(folder_path, file_name), 'r', encoding='utf-8') as file: + with open(os.path.join(folder_path, file_name), "r", encoding="utf-8") as file: response = json.load(file) stream_responses.append(response) @@ -68,8 +68,8 @@ def test_append_response(responses): prompt.append_response(json.dumps(response)) expected_messages = [ - OpenAIMessage(role='assistant', content='Tomorrow.'), - OpenAIMessage(role='assistant', content='Tomorrow!') + OpenAIMessage(role="assistant", content="Tomorrow."), + OpenAIMessage(role="assistant", content="Tomorrow!"), ] assert len(prompt.responses) == len(expected_messages) @@ -85,15 +85,15 @@ def test_messages_empty(): def test_messages_instruct(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") - instruct_message = OpenAIMessage(content='Instructions', role='system') - prompt.append_new(Message.INSTRUCT, 'Instructions') + instruct_message = OpenAIMessage(content="Instructions", role="system") + prompt.append_new(Message.INSTRUCT, "Instructions") assert prompt.messages == [instruct_message.to_dict()] def test_messages_context(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") - context_message = OpenAIMessage(content='Context', role='system') - prompt.append_new(Message.CONTEXT, 'Context') + context_message = OpenAIMessage(content="Context", role="system") + prompt.append_new(Message.CONTEXT, "Context") expected_message = context_message.to_dict() expected_message["content"] = "\n" + context_message.content + "\n" assert prompt.messages == [expected_message] @@ -102,26 +102,26 @@ def test_messages_context(): def test_messages_record(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") with pytest.raises(ValueError): - prompt.append_new(Message.CHAT, 'Record') + prompt.append_new(Message.CHAT, "Record") def test_messages_request(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") - request_message = OpenAIMessage(content='Request', role='user') - prompt.set_request('Request') + request_message = OpenAIMessage(content="Request", role="user") + prompt.set_request("Request") expected_message = request_message.to_dict() assert prompt.messages == [expected_message] def test_messages_combined(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") - instruct_message = OpenAIMessage(content='Instructions', role='system') - context_message = OpenAIMessage(content='Context', role='system') - request_message = OpenAIMessage(content='Request', role='user') + instruct_message = OpenAIMessage(content="Instructions", role="system") + context_message = OpenAIMessage(content="Context", role="system") + request_message = OpenAIMessage(content="Request", role="user") - prompt.append_new(Message.INSTRUCT, 'Instructions') - prompt.append_new(Message.CONTEXT, 'Context') - prompt.set_request('Request') + prompt.append_new(Message.INSTRUCT, "Instructions") + prompt.append_new(Message.CONTEXT, "Context") + prompt.set_request("Request") expected_context_message = context_message.to_dict() expected_context_message["content"] = "\n" + context_message.content + "\n" @@ -132,14 +132,14 @@ def test_messages_combined(): assert prompt.messages == [ instruct_message.to_dict(), expected_request_message, - expected_context_message + expected_context_message, ] def test_messages_invalid_append(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") with pytest.raises(ValueError): - prompt.append_new('invalid', 'Instructions') + prompt.append_new("invalid", "Instructions") def test_messages_invalid_request(): @@ -161,7 +161,7 @@ def test_input_messages(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") messages = [ {"role": "system", "content": "instruction"}, - {"role": "user", "content": "request"} + {"role": "user", "content": "request"}, ] assert prompt.request_tokens == 0 prompt.input_messages(messages) @@ -205,7 +205,7 @@ def test_input_messages(): prompt = OpenAIPrompt("davinci-codex", "John Doe", "john.doe@example.com") messages = [ {"role": "user", "content": "request"}, - {"role": "system", "content": "context1"} + {"role": "system", "content": "context1"}, ] assert prompt.request_tokens == 0 prompt.input_messages(messages) diff --git a/tests/test_prompter.py b/tests/test_prompter.py index 089c704a..9c1ac724 100644 --- a/tests/test_prompter.py +++ b/tests/test_prompter.py @@ -1,4 +1,5 @@ import os + from devchat.engine import Namespace, RecursivePrompter @@ -7,16 +8,16 @@ def test_prompter(tmp_path): prompter = RecursivePrompter(namespace) # Test when there are no 'prompt.txt' files - os.makedirs(os.path.join(tmp_path, 'usr', 'a', 'b', 'c')) - assert prompter.run('a.b.c') == '' + os.makedirs(os.path.join(tmp_path, "usr", "a", "b", "c")) + assert prompter.run("a.b.c") == "" # Test when there is a 'prompt.txt' file in one ancestor - os.makedirs(os.path.join(tmp_path, 'sys', 'a', 'b', 'c')) - with open(os.path.join(tmp_path, 'sys', 'a', 'prompt.txt'), 'w', encoding='utf-8') as file: - file.write('prompt a') - assert prompter.run('a.b.c') == 'prompt a\n' + os.makedirs(os.path.join(tmp_path, "sys", "a", "b", "c")) + with open(os.path.join(tmp_path, "sys", "a", "prompt.txt"), "w", encoding="utf-8") as file: + file.write("prompt a") + assert prompter.run("a.b.c") == "prompt a\n" # Test when there are 'prompt.txt' files in multiple ancestors - with open(os.path.join(tmp_path, 'usr', 'a', 'b', 'prompt.txt'), 'w', encoding='utf-8') as file: - file.write('prompt b') - assert prompter.run('a.b.c') == 'prompt a\nprompt b\n' + with open(os.path.join(tmp_path, "usr", "a", "b", "prompt.txt"), "w", encoding="utf-8") as file: + file.write("prompt b") + assert prompter.run("a.b.c") == "prompt a\nprompt b\n" diff --git a/tests/test_store.py b/tests/test_store.py index d4c5a60b..d24089f2 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,6 +1,8 @@ from typing import Tuple + import pytest -from devchat.openai import OpenAIChatConfig, OpenAIChat + +from devchat.openai import OpenAIChat, OpenAIChatConfig from devchat.store import Store @@ -13,7 +15,7 @@ def create_chat_store(tmp_path) -> Tuple[OpenAIChat, Store]: def _create_response_str(cmpl_id: str, created: int, content: str) -> str: """Create a response string from the given parameters.""" - return f''' + return f""" {{ "id": "{cmpl_id}", "object": "chat.completion", @@ -31,14 +33,17 @@ def _create_response_str(cmpl_id: str, created: int, content: str) -> str: }} ] }} - ''' + """ def test_get_prompt(chat_store): chat, store = chat_store prompt = chat.init_prompt("Where was the 2020 World Series played?") - response_str = _create_response_str("chatcmpl-6p9XYPYSTTRi0xEviKjjilqrWU2Ve", 1577649420, - "The 2020 World Series was played in Arlington, Texas.") + response_str = _create_response_str( + "chatcmpl-6p9XYPYSTTRi0xEviKjjilqrWU2Ve", + 1577649420, + "The 2020 World Series was played in Arlington, Texas.", + ) prompt.set_response(response_str) store.store_prompt(prompt) @@ -52,8 +57,9 @@ def test_select_recent(chat_store): hashes = [] for index in range(5): prompt = chat.init_prompt(f"Question {index}") - response_str = _create_response_str(f"chatcmpl-id{index}", 1577649420 + index, - f"Answer {index}") + response_str = _create_response_str( + f"chatcmpl-id{index}", 1577649420 + index, f"Answer {index}" + ) prompt.set_response(response_str) store.store_prompt(prompt) hashes.append(prompt.hash) @@ -96,8 +102,9 @@ def test_select_recent_with_topic_tree(chat_store): for index in range(1): grandchild_prompt = chat.init_prompt(f"Grandchild question {index}") grandchild_prompt.parent = child_prompt.hash - response_str = _create_response_str(f"chatcmpl-grandchild{index}", 1677649402 + index, - f"Grandchild answer {index}") + response_str = _create_response_str( + f"chatcmpl-grandchild{index}", 1677649402 + index, f"Grandchild answer {index}" + ) grandchild_prompt.set_response(response_str) store.store_prompt(grandchild_prompt) grandchild_hashes.append(grandchild_prompt.hash) @@ -105,7 +112,7 @@ def test_select_recent_with_topic_tree(chat_store): # Test selecting topics topics = store.select_topics(0, 5) assert len(topics) == 1 - assert topics[0]['root_prompt']['hash'] == root_prompt.hash + assert topics[0]["root_prompt"]["hash"] == root_prompt.hash # Test selecting recent prompts within the nested topic recent_prompts = store.select_prompts(1, 3, topic=root_prompt.hash) @@ -134,15 +141,17 @@ def create_prompt_tree(chat_store): # Create and store a grandchild prompt for the child prompt grandchild_prompt = chat.init_prompt("Grandchild question") grandchild_prompt.parent = child_prompt.hash - grandchild_response_str = _create_response_str("chatcmpl-grandchild", 1677649400 + 2, - "Grandchild answer") + grandchild_response_str = _create_response_str( + "chatcmpl-grandchild", 1677649400 + 2, "Grandchild answer" + ) grandchild_prompt.set_response(grandchild_response_str) store.store_prompt(grandchild_prompt) # Create and store another root prompt other_root_prompt = chat.init_prompt("Other root question") - other_root_response_str = _create_response_str("chatcmpl-other-root", 1677649400 + 3, - "Other root answer") + other_root_response_str = _create_response_str( + "chatcmpl-other-root", 1677649400 + 3, "Other root answer" + ) other_root_prompt.set_response(other_root_response_str) store.store_prompt(other_root_prompt) @@ -165,8 +174,8 @@ def test_delete_prompt_grandchild(prompt_tree): # Verify the trees after deletion topics = store.select_topics(0, 5) assert len(topics) == 2 - assert topics[0]['root_prompt']['hash'] == other_root_prompt.hash - assert topics[1]['root_prompt']['hash'] == root_prompt.hash + assert topics[0]["root_prompt"]["hash"] == other_root_prompt.hash + assert topics[1]["root_prompt"]["hash"] == root_prompt.hash def test_delete_prompt_other_root(prompt_tree): @@ -178,4 +187,4 @@ def test_delete_prompt_other_root(prompt_tree): # Verify the trees after deletion topics = store.select_topics(0, 5) assert len(topics) == 1 - assert topics[0]['root_prompt']['hash'] == root_prompt.hash + assert topics[0]["root_prompt"]["hash"] == root_prompt.hash diff --git a/tests/test_utils.py b/tests/test_utils.py index acd84d74..c4736484 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,4 +1,5 @@ import pytest + from devchat.utils import parse_files