Native data AI application framework based on AWEL+AGENT (#1152)

Co-authored-by: Fangyin Cheng <staneyffer@gmail.com>
Co-authored-by: lcx01800250 <lcx01800250@alibaba-inc.com>
Co-authored-by: licunxing <864255598@qq.com>
Co-authored-by: Aralhi <xiaoping0501@gmail.com>
Co-authored-by: xuyuan23 <643854343@qq.com>
Co-authored-by: aries_ckt <916701291@qq.com>
Co-authored-by: hzh97 <2976151305@qq.com>

Author: 8 people

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: python-fmt
         name: Python Format
-        entry: make fmt
+        entry: make fmt-check
         language: system
         exclude: '^dbgpt/app/static/|^web/'
         types: [python]

assets/schema/dbgpt.sql

@@ -262,6 +262,81 @@ CREATE TABLE `gpts_plans` (
   UNIQUE KEY `uk_sub_task` (`conv_id`,`sub_task_num`)
 ) ENGINE=InnoDB AUTO_INCREMENT=19 DEFAULT CHARSET=utf8mb4 COMMENT="gpt plan";
 
+-- dbgpt.dbgpt_serve_flow definition
+CREATE TABLE `dbgpt_serve_flow` (
+  `id` int NOT NULL AUTO_INCREMENT COMMENT 'Auto increment id',
+  `uid` varchar(128) NOT NULL COMMENT 'Unique id',
+  `dag_id` varchar(128) DEFAULT NULL COMMENT 'DAG id',
+  `name` varchar(128) DEFAULT NULL COMMENT 'Flow name',
+  `flow_data` text COMMENT 'Flow data, JSON format',
+  `user_name` varchar(128) DEFAULT NULL COMMENT 'User name',
+  `sys_code` varchar(128) DEFAULT NULL COMMENT 'System code',
+  `gmt_created` datetime DEFAULT NULL COMMENT 'Record creation time',
+  `gmt_modified` datetime DEFAULT NULL COMMENT 'Record update time',
+  `flow_category` varchar(64) DEFAULT NULL COMMENT 'Flow category',
+  `description` varchar(512) DEFAULT NULL COMMENT 'Flow description',
+  `state` varchar(32) DEFAULT NULL COMMENT 'Flow state',
+  `source` varchar(64) DEFAULT NULL COMMENT 'Flow source',
+  `source_url` varchar(512) DEFAULT NULL COMMENT 'Flow source url',
+  `version` varchar(32) DEFAULT NULL COMMENT 'Flow version',
+  `label` varchar(128) DEFAULT NULL COMMENT 'Flow label',
+  `editable` int DEFAULT NULL COMMENT 'Editable, 0: editable, 1: not editable',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_uid` (`uid`),
+  KEY `ix_dbgpt_serve_flow_sys_code` (`sys_code`),
+  KEY `ix_dbgpt_serve_flow_uid` (`uid`),
+  KEY `ix_dbgpt_serve_flow_dag_id` (`dag_id`),
+  KEY `ix_dbgpt_serve_flow_user_name` (`user_name`),
+  KEY `ix_dbgpt_serve_flow_name` (`name`)
+) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+-- dbgpt.gpts_app definition
+CREATE TABLE `gpts_app` (
+  `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
+  `app_code` varchar(255) NOT NULL COMMENT 'Current AI assistant code',
+  `app_name` varchar(255) NOT NULL COMMENT 'Current AI assistant name',
+  `app_describe` varchar(2255) NOT NULL COMMENT 'Current AI assistant describe',
+  `language` varchar(100) NOT NULL COMMENT 'gpts language',
+  `team_mode` varchar(255) NOT NULL COMMENT 'Team work mode',
+  `team_context` text COMMENT 'The execution logic and team member content that teams with different working modes rely on',
+  `user_code` varchar(255) DEFAULT NULL COMMENT 'user code',
+  `sys_code` varchar(255) DEFAULT NULL COMMENT 'system app code',
+  `created_at` datetime DEFAULT NULL COMMENT 'create time',
+  `updated_at` datetime DEFAULT NULL COMMENT 'last update time',
+  `icon` varchar(1024) DEFAULT NULL COMMENT 'app icon, url',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_gpts_app` (`app_name`)
+) ENGINE=InnoDB AUTO_INCREMENT=39 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE `gpts_app_collection` (
+  `id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
+  `app_code` varchar(255) NOT NULL COMMENT 'Current AI assistant code',
+  `user_code` int(11) NOT NULL COMMENT 'user code',
+  `sys_code` varchar(255) NOT NULL COMMENT 'system app code',
+  `created_at` datetime DEFAULT NULL COMMENT 'create time',
+  `updated_at` datetime DEFAULT NULL COMMENT 'last update time',
+  PRIMARY KEY (`id`),
+  KEY `idx_app_code` (`app_code`),
+  KEY `idx_user_code` (`user_code`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4 COMMENT="gpt collections";
+
+-- dbgpt.gpts_app_detail definition
+CREATE TABLE `gpts_app_detail` (
+  `id` int NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
+  `app_code` varchar(255) NOT NULL COMMENT 'Current AI assistant code',
+  `app_name` varchar(255) NOT NULL COMMENT 'Current AI assistant name',
+  `agent_name` varchar(255) NOT NULL COMMENT ' Agent name',
+  `node_id` varchar(255) NOT NULL COMMENT 'Current AI assistant Agent Node id',
+  `resources` text COMMENT 'Agent bind  resource',
+  `prompt_template` text COMMENT 'Agent bind  template',
+  `llm_strategy` varchar(25) DEFAULT NULL COMMENT 'Agent use llm strategy',
+  `llm_strategy_value` text COMMENT 'Agent use llm strategy value',
+  `created_at` datetime DEFAULT NULL COMMENT 'create time',
+  `updated_at` datetime DEFAULT NULL COMMENT 'last update time',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_gpts_app_agent_node` (`app_name`,`agent_name`,`node_id`)
+) ENGINE=InnoDB AUTO_INCREMENT=23 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
 CREATE
 DATABASE IF NOT EXISTS EXAMPLE_1;
 use EXAMPLE_1;

dbgpt/agent/actions/action.py

@@ -0,0 +1,128 @@
+import json
+from abc import ABC
+from typing import (
+    Any,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    get_args,
+    get_origin,
+)
+
+from pydantic import BaseModel
+
+from dbgpt.agent.resource.resource_api import AgentResource, ResourceType
+from dbgpt.agent.resource.resource_loader import ResourceLoader
+from dbgpt.util.json_utils import find_json_objects
+
+T = TypeVar("T", None, BaseModel, List[BaseModel])
+
+
+class ActionOutput(BaseModel):
+    content: str
+    is_exe_success: bool = True
+    view: str = None
+    resource_type: Optional[str] = None
+    resource_value: Optional[Any] = None
+
+    @staticmethod
+    def from_dict(param: Optional[Dict]):
+        if not param:
+            return None
+        return ActionOutput.parse_obj(param)
+
+
+class Action(ABC, Generic[T]):
+    def __init__(self):
+        self.resource_loader: ResourceLoader = None
+
+    def init_resource_loader(self, resource_loader: ResourceLoader):
+        self.resource_loader = resource_loader
+
+    @property
+    def resource_need(self) -> Optional[ResourceType]:
+        return None
+
+    @property
+    def render_protocal(self):
+        raise NotImplementedError("The run method should be implemented in a subclass.")
+
+    def render_prompt(self):
+        if self.render_protocal is None:
+            return None
+        else:
+            return self.render_protocal.render_prompt()
+
+    def _create_example(
+        self,
+        model_type: Union[Type[BaseModel], Type[List[BaseModel]]],
+    ) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        if model_type is None:
+            return None
+        origin = get_origin(model_type)
+        args = get_args(model_type)
+        if origin is None:
+            example = {}
+            for field_name, field in model_type.__fields__.items():
+                field_info = field.field_info
+                if field_info.description:
+                    example[field_name] = field_info.description
+                elif field_info.default:
+                    example[field_name] = field_info.default
+                else:
+                    example[field_name] = ""
+            return example
+        elif origin is list or origin is List:
+            element_type = args[0]
+            if issubclass(element_type, BaseModel):
+                return [self._create_example(element_type)]
+            else:
+                raise TypeError("List elements must be BaseModel subclasses")
+        else:
+            raise ValueError(
+                f"Model type {model_type} is not an instance of BaseModel."
+            )
+
+    @property
+    def out_model_type(self) -> T:
+        return None
+
+    @property
+    def ai_out_schema(self) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        if self.out_model_type is None:
+            return None
+
+        return f"""Please response in the following json format:
+            {json.dumps(self._create_example(self.out_model_type), indent=2, ensure_ascii=False)}
+        Make sure the response is correct json and can be parsed by Python json.loads. 
+        """
+
+    def _ai_mesage_2_json(self, ai_message: str) -> json:
+        json_objects = find_json_objects(ai_message)
+        json_count = len(json_objects)
+        if json_count != 1:
+            raise ValueError("Unable to obtain valid output.")
+        return json_objects[0]
+
+    def _input_convert(self, ai_message: str, cls: Type[T]) -> Union[T, List[T]]:
+        json_result = self._ai_mesage_2_json(ai_message)
+        if get_origin(cls) == list:
+            inner_type = get_args(cls)[0]
+            return [inner_type.parse_obj(item) for item in json_result]
+        else:
+            return cls.parse_obj(json_result)
+
+    async def a_run(
+        self,
+        ai_message: str,
+        resource: Optional[AgentResource] = None,
+        rely_action_out: Optional[ActionOutput] = None,
+        need_vis_render: bool = True,
+        **kwargs,
+    ) -> ActionOutput:
+        raise NotImplementedError("The run method should be implemented in a subclass.")

…ent/agents/expand/rag_assistant_agent.py …/agent/actions/aim_recognition_action.pydbgpt/agent/agents/expand/rag_assistant_agent.py renamed to dbgpt/agent/actions/aim_recognition_action.py dbgpt/agent/agents/expand/rag_assistant_agent.py renamed to dbgpt/agent/actions/aim_recognition_action.py

@@ -0,0 +1,40 @@
+import json
+import logging
+from typing import Any, Dict, List, Optional, Union
+
+from pydantic import BaseModel, Field
+
+from dbgpt.agent.actions.action import Action, ActionOutput, T
+from dbgpt.agent.common.schema import Status
+from dbgpt.agent.plugin.generator import PluginPromptGenerator
+from dbgpt.agent.resource.resource_api import AgentResource, ResourceType
+from dbgpt.agent.resource.resource_plugin_api import ResourcePluginClient
+from dbgpt.vis.tags.vis_plugin import Vis, VisPlugin
+
+logger = logging.getLogger(__name__)
+
+
+class BlankAction(Action):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    @property
+    def resource_need(self) -> Optional[ResourceType]:
+        return None
+
+    @property
+    def render_protocal(self) -> Optional[Vis]:
+        return None
+
+    @property
+    def ai_out_schema(self) -> Union[Dict[str, Any], List[Dict[str, Any]]]:
+        return None
+
+    async def a_run(
+        self,
+        ai_message: str,
+        resource: Optional[AgentResource] = None,
+        rely_action_out: Optional[ActionOutput] = None,
+        need_vis_render: bool = True,
+    ) -> ActionOutput:
+        return ActionOutput(is_exe_success=True, content=ai_message, view=ai_message)

dbgpt/agent/actions/blank_action.py

@@ -0,0 +1,83 @@
+import json
+import logging
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from dbgpt.agent.actions.action import ActionOutput, T
+from dbgpt.agent.resource.resource_api import AgentResource, ResourceType
+from dbgpt.agent.resource.resource_db_api import ResourceDbClient
+from dbgpt.vis.tags.vis_chart import Vis, VisChart
+
+from .action import Action
+
+logger = logging.getLogger(__name__)
+
+
+class SqlInput(BaseModel):
+    display_type: str = Field(
+        ...,
+        description="The chart rendering method selected for SQL. If you don’t know what to output, just output 'response_table' uniformly.",
+    )
+    sql: str = Field(
+        ..., description="Executable sql generated for the current target/problem"
+    )
+    thought: str = Field(..., description="Summary of thoughts to the user")
+
+
+class ChartAction(Action[SqlInput]):
+    def __init__(self):
+        self._render_protocal = VisChart()
+
+    @property
+    def resource_need(self) -> Optional[ResourceType]:
+        return ResourceType.DB
+
+    @property
+    def render_protocal(self) -> Optional[Vis]:
+        return self._render_protocal
+
+    @property
+    def out_model_type(self):
+        return SqlInput
+
+    async def a_run(
+        self,
+        ai_message: str,
+        resource: Optional[AgentResource] = None,
+        rely_action_out: Optional[ActionOutput] = None,
+        need_vis_render: bool = True,
+    ) -> ActionOutput:
+        try:
+            param: SqlInput = self._input_convert(ai_message, SqlInput)
+        except Exception as e:
+            logger.exception(f"str(e)! \n {ai_message}")
+            return ActionOutput(
+                is_exe_success=False,
+                content="The requested correctly structured answer could not be found.",
+            )
+        try:
+            resource_db_client: ResourceDbClient = (
+                self.resource_loader.get_resesource_api(self.resource_need)
+            )
+            if not resource_db_client:
+                raise ValueError(
+                    "There is no implementation class bound to database resource execution！"
+                )
+            data_df = await resource_db_client.a_query_to_df(resource.value, param.sql)
+            view = await self.render_protocal.disply(
+                chart=json.loads(param.json()), data_df=data_df
+            )
+            return ActionOutput(
+                is_exe_success=True,
+                content=param.json(),
+                view=view,
+                resource_type=self.resource_need.value,
+                resource_value=resource.value,
+            )
+        except Exception as e:
+            logger.exception("Check your answers, the sql run failed！")
+            return ActionOutput(
+                is_exe_success=False,
+                content=f"Check your answers, the sql run failed!Reason:{str(e)}",
+            )