feat(model): Support Qwen2.5 VL models (#2743)

Author: fangyinc

packages/dbgpt-core/pyproject.toml

@@ -157,6 +157,9 @@ hf_kimi = [
 hf_qwen3 = [
     "transformers>=4.51.0",
 ]
+hf_qwen2vl = [
+    "transformers>=4.34.0",
+]
 
 [build-system]
 requires = ["hatchling"]

packages/dbgpt-core/src/dbgpt/model/adapter/hf_adapter.py

@@ -553,6 +553,7 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
             and "1.5" in lower_model_name_or_path
             and "moe" not in lower_model_name_or_path
             and "qwen2" not in lower_model_name_or_path
+            and "vl" not in lower_model_name_or_path
         )
 
 
@@ -565,10 +566,12 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
             (
                 "qwen2" in lower_model_name_or_path
                 and "instruct" in lower_model_name_or_path
+                and "vl" not in lower_model_name_or_path
             )
             or (
                 "qwen2.5" in lower_model_name_or_path
                 and "instruct" in lower_model_name_or_path
+                and "vl" not in lower_model_name_or_path
             )
         )
 
@@ -608,6 +611,7 @@ def do_match(self, lower_model_name_or_path: Optional[str] = None):
         return lower_model_name_or_path and (
             "qwen3" in lower_model_name_or_path
             and "base" not in lower_model_name_or_path
+            and "vl" not in lower_model_name_or_path
         )
 
     def check_transformer_version(self, current_version: str) -> None:
@@ -664,6 +668,60 @@ def get_str_prompt(
         return str_prompt
 
 
+class Qwen2VLAdapter(NewHFChatModelAdapter):
+    def check_transformer_version(self, current_version: str) -> None:
+        if not current_version >= "4.37.0":
+            raise ValueError(
+                "Qwen2.5VL model require transformers.__version__>=4.37.0, please "
+                "upgrade your transformers package."
+            )
+
+    def do_match(self, lower_model_name_or_path: Optional[str] = None):
+        return (
+            lower_model_name_or_path
+            and "qwen2" in lower_model_name_or_path
+            and "vl" in lower_model_name_or_path
+            and "instruct" in lower_model_name_or_path
+        )
+
+    def load(self, model_path: str, from_pretrained_kwargs: dict):
+        try:
+            from transformers import (
+                Qwen2_5_VLForConditionalGeneration,
+            )
+        except ImportError as exc:
+            raise ValueError(
+                "Could not import qwen2.5 vl model, please upgrade your "
+                "transformers package to 4.37.0 or later."
+            ) from exc
+
+        logger.info(
+            f"Load model from {model_path}, from_pretrained_kwargs: "
+            f"{from_pretrained_kwargs}"
+        )
+
+        revision = from_pretrained_kwargs.get("revision", "main")
+        trust_remote_code = from_pretrained_kwargs.get(
+            "trust_remote_code", self.trust_remote_code
+        )
+        low_cpu_mem_usage = from_pretrained_kwargs.get("low_cpu_mem_usage", False)
+        if "trust_remote_code" not in from_pretrained_kwargs:
+            from_pretrained_kwargs["trust_remote_code"] = trust_remote_code
+        if "low_cpu_mem_usage" not in from_pretrained_kwargs:
+            from_pretrained_kwargs["low_cpu_mem_usage"] = low_cpu_mem_usage
+
+        tokenizer = self.load_tokenizer(
+            model_path,
+            revision,
+            use_fast=self.use_fast_tokenizer(),
+            trust_remote_code=trust_remote_code,
+        )
+        model = Qwen2_5_VLForConditionalGeneration.from_pretrained(
+            model_path, **from_pretrained_kwargs
+        )
+        return model, tokenizer
+
+
 class QwenOmniAdapter(NewHFChatModelAdapter):
     def do_match(self, lower_model_name_or_path: Optional[str] = None):
         return lower_model_name_or_path and (
@@ -1075,6 +1133,7 @@ def is_reasoning_model(
 register_model_adapter(GLM40414Adapter)
 register_model_adapter(Codegeex4Adapter)
 register_model_adapter(Qwen2Adapter, supported_models=COMMON_HF_QWEN25_MODELS)
+register_model_adapter(Qwen2VLAdapter)
 register_model_adapter(Internlm2Adapter)
 register_model_adapter(DeepseekV3R1Adapter, supported_models=COMMON_HF_DEEPSEEK__MODELS)
 register_model_adapter(KimiVLAdapter)

tests/intetration_tests/datasource/test_conn_mysql.py

@@ -1,21 +1,21 @@
 """
-    Run unit test with command: pytest dbgpt/datasource/rdbms/tests/test_conn_mysql.py
-    docker run -itd --name mysql-test -p 3307:3306 -e MYSQL_ROOT_PASSWORD=12345678 mysql:5.7
-    mysql -h 127.0.0.1 -uroot -p -P3307
-    Enter password:
-    Welcome to the MySQL monitor.  Commands end with ; or \g.
-    Your MySQL connection id is 2
-    Server version: 5.7.41 MySQL Community Server (GPL)
-
-    Copyright (c) 2000, 2023, Oracle and/or its affiliates.
-
-    Oracle is a registered trademark of Oracle Corporation and/or its
-    affiliates. Other names may be trademarks of their respective
-    owners.
-
-    Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
-    
-    > create database test;
+Run unit test with command: pytest dbgpt/datasource/rdbms/tests/test_conn_mysql.py
+docker run -itd --name mysql-test -p 3307:3306 -e MYSQL_ROOT_PASSWORD=12345678 mysql:5.7
+mysql -h 127.0.0.1 -uroot -p -P3307
+Enter password:
+Welcome to the MySQL monitor.  Commands end with ; or \g.
+Your MySQL connection id is 2
+Server version: 5.7.41 MySQL Community Server (GPL)
+
+Copyright (c) 2000, 2023, Oracle and/or its affiliates.
+
+Oracle is a registered trademark of Oracle Corporation and/or its
+affiliates. Other names may be trademarks of their respective
+owners.
+
+Type 'help;' or '\h' for help. Type '\c' to clear the current input statement.
+
+> create database test;
 """
 
 import pytest

tests/intetration_tests/datasource/test_conn_oracle.py

@@ -116,4 +116,4 @@ def test_get_database_lists(db):
         pdb_names = [name[0] for name in databases[1:]]
     else:
         pdb_names = ["ORCL"]
-    assert any(name in ("ORCLPDB", "ORCL") for name in pdb_names)
+    assert any(name in ("ORCLPDB", "ORCL") for name in pdb_names)