From c09224023dc1e6d6d1e416fefd6786050369e958 Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Fri, 11 Jul 2025 17:39:36 +0900 Subject: [PATCH 01/14] Support json response format Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- tensorrt_llm/serve/openai_protocol.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 84594cd473f9..559a6fc54e85 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -53,7 +53,8 @@ class StructuralTag(OpenAIBaseModel): class ResponseFormat(OpenAIBaseModel): # type must be "json_object" or "text" or "structural_tag" - type: Literal["text", "json_object", "structural_tag"] + type: Literal["json", "text", "json_object", "structural_tag"] + schema: Optional[Union[str, BaseModel, dict]] = None structures: Optional[List[StructuralTag]] = None triggers: Optional[List[str]] = None @@ -140,6 +141,12 @@ def _response_format_to_guided_decoding_params( ) -> Optional[GuidedDecodingParams]: if response_format is None: return None + elif response_format.type == "json": + if response_format.schema is None: + raise ValueError( + "The 'schema' field is required when response_format.type is 'json'." + ) + return GuidedDecodingParams(json=response_format.schema) elif response_format.type == "text": return None elif response_format.type == "json_object": From 1eca8763eb906b6a242632ba154033c5ecd22a81 Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Fri, 11 Jul 2025 18:03:10 +0900 Subject: [PATCH 02/14] support dict only Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- tensorrt_llm/serve/openai_protocol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 559a6fc54e85..dcb5cdd754f3 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -54,7 +54,7 @@ class StructuralTag(OpenAIBaseModel): class ResponseFormat(OpenAIBaseModel): # type must be "json_object" or "text" or "structural_tag" type: Literal["json", "text", "json_object", "structural_tag"] - schema: Optional[Union[str, BaseModel, dict]] = None + schema: Optional[dict] = None structures: Optional[List[StructuralTag]] = None triggers: Optional[List[str]] = None From 90e488edacb45b642c460830874604ae0aa4060e Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:17:48 +0900 Subject: [PATCH 03/14] refactor: reorder decoding types Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- tensorrt_llm/serve/openai_protocol.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index dcb5cdd754f3..b4d1fbdd9184 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -52,8 +52,8 @@ class StructuralTag(OpenAIBaseModel): class ResponseFormat(OpenAIBaseModel): - # type must be "json_object" or "text" or "structural_tag" - type: Literal["json", "text", "json_object", "structural_tag"] + # type must be one of "text", "json", "json_object", or "structural_tag" + type: Literal["text", "json", "json_object", "structural_tag"] schema: Optional[dict] = None structures: Optional[List[StructuralTag]] = None triggers: Optional[List[str]] = None @@ -141,14 +141,14 @@ def _response_format_to_guided_decoding_params( ) -> Optional[GuidedDecodingParams]: if response_format is None: return None + elif response_format.type == "text": + return None elif response_format.type == "json": if response_format.schema is None: raise ValueError( "The 'schema' field is required when response_format.type is 'json'." ) return GuidedDecodingParams(json=response_format.schema) - elif response_format.type == "text": - return None elif response_format.type == "json_object": return GuidedDecodingParams(json_object=True) elif response_format.type == "structural_tag": From cfb833f9c4f356c870c2a1fe6b5a4d6ed405e239 Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:38:15 +0900 Subject: [PATCH 04/14] Add _test_openai_chat_json.py Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- .../llmapi/apps/_test_openai_chat_json.py | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 tests/unittest/llmapi/apps/_test_openai_chat_json.py diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_json.py b/tests/unittest/llmapi/apps/_test_openai_chat_json.py new file mode 100644 index 000000000000..f09a15d9e5e9 --- /dev/null +++ b/tests/unittest/llmapi/apps/_test_openai_chat_json.py @@ -0,0 +1,131 @@ +# Adapted from +# https://github.com/vllm-project/vllm/blob/aae6927be06dedbda39c6b0c30f6aa3242b84388/tests/entrypoints/openai/test_chat.py +import os +import tempfile + +import openai +import pytest +import yaml + +from ..test_llm import get_model_path, similar +from .openai_server import RemoteOpenAIServer + +pytestmark = pytest.mark.threadleak(enabled=False) + + +@pytest.fixture(scope="module", ids=["TinyLlama-1.1B-Chat"]) +def model_name(): + return "llama-3.1-model/Llama-3.1-8B-Instruct" + + +@pytest.fixture(scope="module") +def temp_extra_llm_api_options_file(request): + temp_dir = tempfile.gettempdir() + temp_file_path = os.path.join(temp_dir, "extra_llm_api_options.yaml") + try: + extra_llm_api_options_dict = { + "guided_decoding_backend": "xgrammar", + "disable_overlap_scheduler": True, # Guided decoding is not supported with overlap scheduler + } + + with open(temp_file_path, "w") as f: + yaml.dump(extra_llm_api_options_dict, f) + + yield temp_file_path + finally: + if os.path.exists(temp_file_path): + os.remove(temp_file_path) + + +@pytest.fixture(scope="module") +def server(model_name: str, temp_extra_llm_api_options_file: str): + model_path = get_model_path(model_name) + args = [ + "--backend", "pytorch", "--extra_llm_api_options", + temp_extra_llm_api_options_file + ] + with RemoteOpenAIServer(model_path, args) as remote_server: + yield remote_server + + +@pytest.fixture(scope="module") +def client(server: RemoteOpenAIServer): + return server.get_client() + + +@pytest.fixture(scope="module") +def async_client(server: RemoteOpenAIServer): + return server.get_async_client() + + +@pytest.fixture(scope="module") +def user_profile_schema(): + """Provides a sample JSON schema for a user profile.""" + return { + "type": "object", + "properties": { + "name": {"type": "string", "description": "The full name of the user."}, + "age": {"type": "integer", "description": "The age of the user, in years."}, + }, + "required": ["name", "age"], + } + + +def test_chat_json_schema(client: openai.OpenAI, model_name: str): + """ + Tests the `json` response format in a multi-turn synchronous conversation. + Adapted from https://github.com/vllm-project/vllm/blob/aae6927be06dedbda39c6b0c30f6aa3242b84388/tests/entrypoints/openai/test_chat.py#L413 + """ + + def _create_and_validate_response(messages: list[dict[str, Any]]) -> dict[str, any]: + chat_completion = client.chat.completions.create( + model=model_name, + messages=messages, + max_tokens=1000, + temperature=0.0, + response_format={"type": "json", "schema": user_profile_schema}, + ) + message = chat_completion.choices[0].message + assert message.content is not None + + try: + message_json = json.loads(message.content) + except json.JSONDecodeError: + pytest.fail( + f"The output was not a valid JSON string. Output: {output_text}" + ) + + jsonschema.validate(instance=message_json, schema=user_profile_schema) + return message_json + + messages = [ + {"role": "system", "content": "you are a helpful assistant"}, + { + "role": "user", + "content": f"Give an example JSON for an employee profile that " + f"fits this schema: {user_profile_schema}", + }, + ] + + first_json = _create_and_validate_response(messages) + + messages.extend( + [ + { + "role": "assistant", + "content": first_message.content, + }, + { + "role": "user", + "content": "Give me another one with a different name and age.", + }, + ] + ) + second_json = _create_and_validate_response(messages) + + assert ( + first_json["name"] != second_json["name"] + ), "The model should have generated a different name in the second turn." + assert ( + first_json["age"] != second_json["age"] + ), "The model should have generated a different age in the second turn." From 864d2e0075ec97730b5e3b3e8346aad8da7482ab Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:40:36 +0900 Subject: [PATCH 05/14] Update test_e2e.py Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- tests/integration/defs/test_e2e.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 7a3becba969f..8de25bcbe7dc 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1456,6 +1456,15 @@ def test_openai_chat_structural_tag_example(llm_venv): ]) +def test_openai_chat_json_example(llm_venv): + test_root = unittest_path() / "llmapi" / "apps" + + llm_venv.run_cmd([ + "-m", "pytest", + str(test_root / "_test_openai_chat_json.py") + ]) + + @pytest.mark.skip_less_device(2) @pytest.mark.skip_less_device_memory(40000) def test_openai_multi_chat_example(llm_root, llm_venv): From ef9b27fcfa9557af1a461b2e2b724c359b91db82 Mon Sep 17 00:00:00 2001 From: noiji <52301388+noiji@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:41:52 +0900 Subject: [PATCH 06/14] Update l0_a10.yml Signed-off-by: noiji <52301388+noiji@users.noreply.github.com> Signed-off-by: noiji --- tests/integration/test_lists/test-db/l0_a10.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_lists/test-db/l0_a10.yml b/tests/integration/test_lists/test-db/l0_a10.yml index 2f63ab45f3aa..e98dd1f5640a 100644 --- a/tests/integration/test_lists/test-db/l0_a10.yml +++ b/tests/integration/test_lists/test-db/l0_a10.yml @@ -22,6 +22,7 @@ l0_a10: - disaggregated/test_disaggregated.py::test_disaggregated_mixed[TinyLlama-1.1B-Chat-v1.0] - disaggregated/test_disaggregated.py::test_disaggregated_overlap[TinyLlama-1.1B-Chat-v1.0] - test_e2e.py::test_openai_chat_structural_tag_example + - test_e2e.py::test_openai_chat_json_example - test_e2e.py::test_openai_chat_multimodal_example - test_e2e.py::test_openai_lora - test_e2e.py::test_trtllm_serve_multimodal_example From d304509a7028566b7b355dc6c250adee4069e5a8 Mon Sep 17 00:00:00 2001 From: noiji Date: Tue, 15 Jul 2025 18:55:34 +0900 Subject: [PATCH 07/14] apply formatter Signed-off-by: noiji --- tests/integration/defs/test_e2e.py | 7 +-- .../llmapi/apps/_test_openai_chat_json.py | 56 ++++++++++++------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/tests/integration/defs/test_e2e.py b/tests/integration/defs/test_e2e.py index 8de25bcbe7dc..7bc2e5adf51b 100644 --- a/tests/integration/defs/test_e2e.py +++ b/tests/integration/defs/test_e2e.py @@ -1459,10 +1459,9 @@ def test_openai_chat_structural_tag_example(llm_venv): def test_openai_chat_json_example(llm_venv): test_root = unittest_path() / "llmapi" / "apps" - llm_venv.run_cmd([ - "-m", "pytest", - str(test_root / "_test_openai_chat_json.py") - ]) + llm_venv.run_cmd( + ["-m", "pytest", + str(test_root / "_test_openai_chat_json.py")]) @pytest.mark.skip_less_device(2) diff --git a/tests/unittest/llmapi/apps/_test_openai_chat_json.py b/tests/unittest/llmapi/apps/_test_openai_chat_json.py index f09a15d9e5e9..3cec170b0450 100644 --- a/tests/unittest/llmapi/apps/_test_openai_chat_json.py +++ b/tests/unittest/llmapi/apps/_test_openai_chat_json.py @@ -7,7 +7,7 @@ import pytest import yaml -from ..test_llm import get_model_path, similar +from ..test_llm import get_model_path from .openai_server import RemoteOpenAIServer pytestmark = pytest.mark.threadleak(enabled=False) @@ -25,7 +25,8 @@ def temp_extra_llm_api_options_file(request): try: extra_llm_api_options_dict = { "guided_decoding_backend": "xgrammar", - "disable_overlap_scheduler": True, # Guided decoding is not supported with overlap scheduler + "disable_overlap_scheduler": + True, # Guided decoding is not supported with overlap scheduler } with open(temp_file_path, "w") as f: @@ -64,8 +65,14 @@ def user_profile_schema(): return { "type": "object", "properties": { - "name": {"type": "string", "description": "The full name of the user."}, - "age": {"type": "integer", "description": "The age of the user, in years."}, + "name": { + "type": "string", + "description": "The full name of the user." + }, + "age": { + "type": "integer", + "description": "The age of the user, in years." + }, }, "required": ["name", "age"], } @@ -77,13 +84,17 @@ def test_chat_json_schema(client: openai.OpenAI, model_name: str): Adapted from https://github.com/vllm-project/vllm/blob/aae6927be06dedbda39c6b0c30f6aa3242b84388/tests/entrypoints/openai/test_chat.py#L413 """ - def _create_and_validate_response(messages: list[dict[str, Any]]) -> dict[str, any]: + def _create_and_validate_response( + messages: list[dict[str, Any]]) -> dict[str, any]: chat_completion = client.chat.completions.create( model=model_name, messages=messages, max_tokens=1000, temperature=0.0, - response_format={"type": "json", "schema": user_profile_schema}, + response_format={ + "type": "json", + "schema": user_profile_schema + }, ) message = chat_completion.choices[0].message assert message.content is not None @@ -99,28 +110,31 @@ def _create_and_validate_response(messages: list[dict[str, Any]]) -> dict[str, a return message_json messages = [ - {"role": "system", "content": "you are a helpful assistant"}, { - "role": "user", - "content": f"Give an example JSON for an employee profile that " + "role": "system", + "content": "you are a helpful assistant" + }, + { + "role": + "user", + "content": + f"Give an example JSON for an employee profile that " f"fits this schema: {user_profile_schema}", }, ] first_json = _create_and_validate_response(messages) - messages.extend( - [ - { - "role": "assistant", - "content": first_message.content, - }, - { - "role": "user", - "content": "Give me another one with a different name and age.", - }, - ] - ) + messages.extend([ + { + "role": "assistant", + "content": first_message.content, + }, + { + "role": "user", + "content": "Give me another one with a different name and age.", + }, + ]) second_json = _create_and_validate_response(messages) assert ( From 834821d332361b37b8bd1c4183830fda1ab28e98 Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Fri, 18 Jul 2025 15:49:15 -0700 Subject: [PATCH 08/14] Update openai_protocol.py adding the changes to support the json_schema as one of the supported type Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_protocol.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 84594cd473f9..bd45d96102e8 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -52,10 +52,11 @@ class StructuralTag(OpenAIBaseModel): class ResponseFormat(OpenAIBaseModel): - # type must be "json_object" or "text" or "structural_tag" - type: Literal["text", "json_object", "structural_tag"] + # type must be "json_object" or "text" or "structural_tag" or "json_schema" + type: Literal["text", "json_object", "structural_tag", "json_schema"] structures: Optional[List[StructuralTag]] = None triggers: Optional[List[str]] = None + json_schema: Optional[Dict[str, Any]] = None class DisaggregatedParams(OpenAIBaseModel): @@ -144,6 +145,8 @@ def _response_format_to_guided_decoding_params( return None elif response_format.type == "json_object": return GuidedDecodingParams(json_object=True) + elif response_format.type == "json_schema": + return GuidedDecodingParams(json=response_format.json_schema) elif response_format.type == "structural_tag": return GuidedDecodingParams( structural_tag=response_format.model_dump_json(by_alias=True, @@ -205,7 +208,7 @@ class CompletionRequest(OpenAIBaseModel): default=None, description= ("Similar to chat completion, this parameter specifies the format of " - "output. {'type': 'json_object'}, {'type': 'text' }, {'type': 'structural_tag'} are " + "output. {'type': 'json_object'}, {'type': 'text' }, {'type': 'structural_tag'}, {'type': 'json_schema'} are " "supported."), ) From 52f833a2222d1dc1c72878e2fe7377a79fd679d0 Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Fri, 18 Jul 2025 16:23:02 -0700 Subject: [PATCH 09/14] Update openai_server.py adding flags related to the lora_request else it will give 400 request code Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 02d77232ab26..035e68e1d294 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -288,6 +288,11 @@ async def create_chat_response( if request.stream else chat_response_post_processor, postproc_args=postproc_args, ) + + lora_request = None + if hasattr(request, 'lora_request') and request.lora_request: + lora_request = request.lora_req + promise = self.llm.generate_async( inputs=prompt, From 30096621761e617d6e7f06c93f39a9bf83be2a07 Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Fri, 18 Jul 2025 17:06:25 -0700 Subject: [PATCH 10/14] Update openai_server.py fixing typo with `lora_request` Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 035e68e1d294..830a184139af 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -291,7 +291,7 @@ async def create_chat_response( lora_request = None if hasattr(request, 'lora_request') and request.lora_request: - lora_request = request.lora_req + lora_request = request.lora_reqeuest promise = self.llm.generate_async( From 02e62f2998d6a58c095bfc5e45b83026f72bb27a Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Mon, 21 Jul 2025 09:49:01 -0700 Subject: [PATCH 11/14] Update openai_server.py Fixing the `lora_request` typo Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 830a184139af..e7701803abed 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -291,7 +291,7 @@ async def create_chat_response( lora_request = None if hasattr(request, 'lora_request') and request.lora_request: - lora_request = request.lora_reqeuest + lora_request = request.lora_request promise = self.llm.generate_async( From c709b283ce680a509d5118a0e287098229543a75 Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Tue, 22 Jul 2025 00:01:28 -0700 Subject: [PATCH 12/14] Update openai_server.py removing the `lora_request` as the latest main branch contains this defined. Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- tensorrt_llm/serve/openai_server.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index e7701803abed..02d77232ab26 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -288,11 +288,6 @@ async def create_chat_response( if request.stream else chat_response_post_processor, postproc_args=postproc_args, ) - - lora_request = None - if hasattr(request, 'lora_request') and request.lora_request: - lora_request = request.lora_request - promise = self.llm.generate_async( inputs=prompt, From 93ba7d5e052207badf95a63facf494a9fc9c3e2e Mon Sep 17 00:00:00 2001 From: mayani-nv <67936769+mayani-nv@users.noreply.github.com> Date: Tue, 22 Jul 2025 00:10:57 -0700 Subject: [PATCH 13/14] Create _test_openai_json_schema.py Adding the unit test for the json_schema support in xgrammar Signed-off-by: mayani-nv <67936769+mayani-nv@users.noreply.github.com> --- .../llmapi/apps/_test_openai_json_schema.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 tests/unittest/llmapi/apps/_test_openai_json_schema.py diff --git a/tests/unittest/llmapi/apps/_test_openai_json_schema.py b/tests/unittest/llmapi/apps/_test_openai_json_schema.py new file mode 100644 index 000000000000..bcdaf9bee50c --- /dev/null +++ b/tests/unittest/llmapi/apps/_test_openai_json_schema.py @@ -0,0 +1,103 @@ +import os +import tempfile + +import openai +import pytest +import yaml +from pydantic import BaseModel, Field + +from ..test_llm import get_model_path +from .openai_server import RemoteOpenAIServer + +pytestmark = pytest.mark.threadleak(enabled=False) + + +@pytest.fixture(scope="module", ids=["TinyLlama-1.1B-Chat"]) +def model_name(): + return "llama-3.1-model/Llama-3.1-8B-Instruct" + + +@pytest.fixture(scope="module") +def temp_extra_llm_api_options_file(request): + temp_dir = tempfile.gettempdir() + temp_file_path = os.path.join(temp_dir, "extra_llm_api_options.yaml") + try: + extra_llm_api_options_dict = {"guided_decoding_backend": "xgrammar"} + + with open(temp_file_path, 'w') as f: + yaml.dump(extra_llm_api_options_dict, f) + + yield temp_file_path + finally: + if os.path.exists(temp_file_path): + os.remove(temp_file_path) + + +@pytest.fixture(scope="module") +def server(model_name: str, temp_extra_llm_api_options_file: str): + model_path = get_model_path(model_name) + args = [ + "--backend", "pytorch", "--extra_llm_api_options", + temp_extra_llm_api_options_file + ] + with RemoteOpenAIServer(model_path, args) as remote_server: + yield remote_server + + +@pytest.fixture(scope="module") +def client(server: RemoteOpenAIServer): + return server.get_client() + + +@pytest.fixture(scope="module") +def async_client(server: RemoteOpenAIServer): + return server.get_async_client() + + +@pytest.fixture(scope="module") +def capital_info_model(): + + class CapitalInfo(BaseModel): + name: str = Field(..., + pattern=r"^\w+$", + description="The name of the capital city") + population: int = Field(..., + description="The population of the capital city") + + return CapitalInfo + + +def test_chat_json_schema(client: openai.OpenAI, model_name: str, + capital_info_model): + + CapitalInfo = capital_info_model + messages = [{ + "role": + "user", + "content": + "Please generate the information of the capital of France in the JSON format. ", + }, ] + + chat_completion = client.chat.completions.create( + model=model_name, + messages=messages, + response_format={ + "type": "json_schema", + "json_schema": CapitalInfo.model_json_schema(), + }, + temperature=0.7, + max_completion_tokens=100, + ) + + assert chat_completion.id is not None + assert len(chat_completion.choices) == 1 + message = chat_completion.choices[0].message + assert message.content is not None + assert message.role == "assistant" + + capital_info = CapitalInfo.model_validate_json(message.content) + + assert isinstance(capital_info, CapitalInfo) + assert capital_info.name == "Paris" + assert isinstance(capital_info.population, int) + assert capital_info.population > 0 From cbb407e625f41d8141b75d08ad5aaa684678280e Mon Sep 17 00:00:00 2001 From: mayani-nv Date: Thu, 24 Jul 2025 00:36:18 +0000 Subject: [PATCH 14/14] refactoring code to use PR 5957 --- tensorrt_llm/serve/openai_protocol.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 95c6b64c0a30..f381757d448c 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -57,7 +57,6 @@ class ResponseFormat(OpenAIBaseModel): schema: Optional[dict] = None structures: Optional[List[StructuralTag]] = None triggers: Optional[List[str]] = None - json_schema: Optional[Dict[str, Any]] = None class DisaggregatedParams(OpenAIBaseModel): @@ -213,7 +212,7 @@ class CompletionRequest(OpenAIBaseModel): default=None, description= ("Similar to chat completion, this parameter specifies the format of " - "output. {'type': 'json_object'}, {'type': 'text' }, {'type': 'structural_tag'}, {'type': 'json_schema'} are " + "output. {'type': 'json_object'}, {'type': 'text' }, {'type': 'structural_tag'}, {'type': 'json'} are " "supported."), )