From db23f1802963501dce44653c4d56b846387203a7 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Fri, 22 Aug 2025 07:29:37 +0000 Subject: [PATCH 01/19] add debug cdoes for evaluation Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 ++++ tests/integration/defs/accuracy/test_llm_api_pytorch.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index c77e3be4479e..eea13acdf793 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -398,6 +398,10 @@ def evaluate(self, system_instruction=self.system_prompt) # Normalize scores to range 0~100 scores = results["results"][self.task_name] + if self.task_name == "gsm8k": + print(f"scores: {scores}, results: {results}") + import sys + sys.stdout.flush() for metric in scores.keys(): if isinstance(scores[metric], (float, int)): scores[metric] *= 100 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 0390c97e64ce..c962ab076dce 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2343,8 +2343,8 @@ def test_nvfp4( **pytorch_config, enable_attention_dp=attention_dp, max_batch_size=32) as llm: - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) + # task = MMLU(self.MODEL_NAME) + # task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.evaluate(llm) From 4637e5affb49273c94250e0b547a067dcb42b352 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Fri, 22 Aug 2025 12:10:37 +0000 Subject: [PATCH 02/19] add debug info Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tests/integration/defs/accuracy/accuracy_core.py | 2 +- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index 06303886292b..e4f4343e248e 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -297,7 +297,7 @@ class GSM8K(AccuracyTask): ALPHA = 0.05 BETA = 0.2 SIGMA = 50 - NUM_SAMPLES = 1319 # Full sample + NUM_SAMPLES = 2 # Full sample MAX_INPUT_LEN = 4096 MAX_OUTPUT_LEN = 256 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index c962ab076dce..6f6f753de331 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2347,6 +2347,7 @@ def test_nvfp4( # task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + assert False def test_eagle3(self): pytorch_config = dict( From d197ef5ffc74bf0685932b73db03663de9b5255a Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Sat, 23 Aug 2025 10:14:38 +0000 Subject: [PATCH 03/19] set apply_chat_template as false for gsm8k Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tests/integration/defs/accuracy/accuracy_core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index e4f4343e248e..066cb617ff48 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -303,7 +303,8 @@ class GSM8K(AccuracyTask): MAX_OUTPUT_LEN = 256 EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8K - EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0) + EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0, + apply_chat_template=False) EVALUATE_KWARGS = dict(scores_filter=None) From b9a8c8817fb808520f548cedd41f57e3a48bd214 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Sat, 23 Aug 2025 10:16:04 +0000 Subject: [PATCH 04/19] revert changes of debugging Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 ---- tests/integration/defs/accuracy/accuracy_core.py | 2 +- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 5 ++--- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index eea13acdf793..c77e3be4479e 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -398,10 +398,6 @@ def evaluate(self, system_instruction=self.system_prompt) # Normalize scores to range 0~100 scores = results["results"][self.task_name] - if self.task_name == "gsm8k": - print(f"scores: {scores}, results: {results}") - import sys - sys.stdout.flush() for metric in scores.keys(): if isinstance(scores[metric], (float, int)): scores[metric] *= 100 diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index 066cb617ff48..7325f3c43fc4 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -297,7 +297,7 @@ class GSM8K(AccuracyTask): ALPHA = 0.05 BETA = 0.2 SIGMA = 50 - NUM_SAMPLES = 2 # Full sample + NUM_SAMPLES = 1319 # Full sample MAX_INPUT_LEN = 4096 MAX_OUTPUT_LEN = 256 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 6f6f753de331..0390c97e64ce 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2343,11 +2343,10 @@ def test_nvfp4( **pytorch_config, enable_attention_dp=attention_dp, max_batch_size=32) as llm: - # task = MMLU(self.MODEL_NAME) - # task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.evaluate(llm) - assert False def test_eagle3(self): pytorch_config = dict( From 3bd3c24f2fb599008f07a3e44fb0d6de6d75b923 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Sat, 23 Aug 2025 14:47:29 +0000 Subject: [PATCH 05/19] fix pre-commit issue Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tests/integration/defs/accuracy/accuracy_core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index 7325f3c43fc4..7d5f939a24cf 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -303,7 +303,8 @@ class GSM8K(AccuracyTask): MAX_OUTPUT_LEN = 256 EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8K - EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0, + EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, + random_seed=0, apply_chat_template=False) EVALUATE_KWARGS = dict(scores_filter=None) From 464096d98a9e986337e8a3912acdbba9c315f017 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Sat, 23 Aug 2025 16:07:10 -0700 Subject: [PATCH 06/19] add debug info again since apply_chat_template=False does not work Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 ++++ tests/integration/defs/accuracy/accuracy_core.py | 2 +- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index c77e3be4479e..eea13acdf793 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -398,6 +398,10 @@ def evaluate(self, system_instruction=self.system_prompt) # Normalize scores to range 0~100 scores = results["results"][self.task_name] + if self.task_name == "gsm8k": + print(f"scores: {scores}, results: {results}") + import sys + sys.stdout.flush() for metric in scores.keys(): if isinstance(scores[metric], (float, int)): scores[metric] *= 100 diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index 7d5f939a24cf..e4fc6be5c3c0 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -297,7 +297,7 @@ class GSM8K(AccuracyTask): ALPHA = 0.05 BETA = 0.2 SIGMA = 50 - NUM_SAMPLES = 1319 # Full sample + NUM_SAMPLES = 2 # Full sample MAX_INPUT_LEN = 4096 MAX_OUTPUT_LEN = 256 diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 0390c97e64ce..0738c9d1d13b 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2347,6 +2347,7 @@ def test_nvfp4( task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.evaluate(llm) + assert False def test_eagle3(self): pytorch_config = dict( From 24eaaf3cd2781245addd2756858bdb3d4cbdd79f Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Sun, 24 Aug 2025 13:02:40 +0000 Subject: [PATCH 07/19] force to disable the apply_chat_tempalte Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tests/integration/defs/accuracy/test_llm_api_pytorch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 0738c9d1d13b..1afa2cc6f919 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2346,6 +2346,7 @@ def test_nvfp4( task = MMLU(self.MODEL_NAME) task.evaluate(llm) task = GSM8K(self.MODEL_NAME) + task.apply_chat_template = False task.evaluate(llm) assert False From 747a491c1518be1c86fb0c5ca6e629502de635d4 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 00:14:16 +0000 Subject: [PATCH 08/19] print scores_filter Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index eea13acdf793..9087cc8463fd 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -400,6 +400,7 @@ def evaluate(self, scores = results["results"][self.task_name] if self.task_name == "gsm8k": print(f"scores: {scores}, results: {results}") + print(f"scores_filter: {scores_filter}") import sys sys.stdout.flush() for metric in scores.keys(): From 64f8c9d82e06a1a07ad2b9c3cfa733d29996c54b Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 02:24:21 +0000 Subject: [PATCH 09/19] add debug info Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/interface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorrt_llm/evaluate/interface.py b/tensorrt_llm/evaluate/interface.py index 3a038919ced3..699ef6505e6f 100644 --- a/tensorrt_llm/evaluate/interface.py +++ b/tensorrt_llm/evaluate/interface.py @@ -86,6 +86,8 @@ def evaluate(self, outputs, references, auxiliaries = [], [], [] for prompt, sampling_args, reference, *aux in tqdm( self.generate_samples(), desc="Submitting requests"): + print(f"interface.py::evaluate: self.apply_chat_template: {self.apply_chat_template}") + print(f"interface.py::evaluate: prompt: {prompt}") if self.apply_chat_template: prompt = self.do_apply_chat_template(llm, prompt) sampling_params = self._get_sampline_params(sampling_params, From 8658f29f1b686a240f4f0d701f3da59dc43ad9ca Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 03:04:36 +0000 Subject: [PATCH 10/19] fix pre-commit Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/interface.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/evaluate/interface.py b/tensorrt_llm/evaluate/interface.py index 699ef6505e6f..53b13d428d7d 100644 --- a/tensorrt_llm/evaluate/interface.py +++ b/tensorrt_llm/evaluate/interface.py @@ -86,7 +86,9 @@ def evaluate(self, outputs, references, auxiliaries = [], [], [] for prompt, sampling_args, reference, *aux in tqdm( self.generate_samples(), desc="Submitting requests"): - print(f"interface.py::evaluate: self.apply_chat_template: {self.apply_chat_template}") + print( + f"interface.py::evaluate: self.apply_chat_template: {self.apply_chat_template}" + ) print(f"interface.py::evaluate: prompt: {prompt}") if self.apply_chat_template: prompt = self.do_apply_chat_template(llm, prompt) From 9462052f1c3f3c6791b44343e4eed188d89e2ba3 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 06:04:28 +0000 Subject: [PATCH 11/19] adjust test Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/interface.py | 4 ---- tensorrt_llm/evaluate/lm_eval.py | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorrt_llm/evaluate/interface.py b/tensorrt_llm/evaluate/interface.py index 53b13d428d7d..3a038919ced3 100644 --- a/tensorrt_llm/evaluate/interface.py +++ b/tensorrt_llm/evaluate/interface.py @@ -86,10 +86,6 @@ def evaluate(self, outputs, references, auxiliaries = [], [], [] for prompt, sampling_args, reference, *aux in tqdm( self.generate_samples(), desc="Submitting requests"): - print( - f"interface.py::evaluate: self.apply_chat_template: {self.apply_chat_template}" - ) - print(f"interface.py::evaluate: prompt: {prompt}") if self.apply_chat_template: prompt = self.do_apply_chat_template(llm, prompt) sampling_params = self._get_sampline_params(sampling_params, diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 9087cc8463fd..f51ecfa7dea1 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -389,6 +389,7 @@ def evaluate(self, scores_filter: str = None) -> float: import lm_eval lm_cls = MultimodalLmEvalWrapper if self.MULTIMODAL else LmEvalWrapper + print(f"self.apply_chat_template: {self.apply_chat_template}") results = lm_eval.evaluate( lm=lm_cls(llm, sampling_params, streaming), task_dict=self.task_dict, From 2a154ec0e4b92646635adee6b5b6f585db17b091 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 02:31:22 -0700 Subject: [PATCH 12/19] add more debug info Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 1 + tests/integration/defs/accuracy/test_llm_api_pytorch.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index f51ecfa7dea1..3d03107e2b7e 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -311,6 +311,7 @@ def __init__(self, ) from e import lm_eval.tasks self.MULTIMODAL = is_multimodal + print(f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}") if self.MULTIMODAL: apply_chat_template = True logger.info( diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 1afa2cc6f919..16ad0852f4a2 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2343,10 +2343,11 @@ def test_nvfp4( **pytorch_config, enable_attention_dp=attention_dp, max_batch_size=32) as llm: - task = MMLU(self.MODEL_NAME) - task.evaluate(llm) + # task = MMLU(self.MODEL_NAME) + # task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.apply_chat_template = False + print(f"task.apply_chat_template: {task.apply_chat_template}") task.evaluate(llm) assert False From 5fa36ea0edf590aafe8f6c9febae91de1af672e5 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 12:33:29 +0000 Subject: [PATCH 13/19] fix bug of pre-commit Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 3d03107e2b7e..b31fd36318a6 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -390,7 +390,9 @@ def evaluate(self, scores_filter: str = None) -> float: import lm_eval lm_cls = MultimodalLmEvalWrapper if self.MULTIMODAL else LmEvalWrapper - print(f"self.apply_chat_template: {self.apply_chat_template}") + print( + f"LmEvalEvaluator::evaluate: apply_chat_template: {self.apply_chat_template}" + ) results = lm_eval.evaluate( lm=lm_cls(llm, sampling_params, streaming), task_dict=self.task_dict, From f0d34d2350d827371927568e62e4f5e92da43a04 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 06:37:13 -0700 Subject: [PATCH 14/19] fix bug of pre-commit Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index b31fd36318a6..173fd3519041 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -311,7 +311,9 @@ def __init__(self, ) from e import lm_eval.tasks self.MULTIMODAL = is_multimodal - print(f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}") + print( + f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}" + ) if self.MULTIMODAL: apply_chat_template = True logger.info( From 4e70488354b431431f2cf44d1e275c206b0edccd Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Mon, 25 Aug 2025 09:07:46 -0700 Subject: [PATCH 15/19] add assertion for debug case Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 173fd3519041..e6d29126be21 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -314,6 +314,8 @@ def __init__(self, print( f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}" ) + if apply_chat_template: + assert False if self.MULTIMODAL: apply_chat_template = True logger.info( From a6f5a48fa6edad6a5b311c3fd388ad0e13dba3b3 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Tue, 26 Aug 2025 00:57:47 +0000 Subject: [PATCH 16/19] add traceback into LmEvalEvaluator Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index e6d29126be21..84bce6eee441 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -314,8 +314,8 @@ def __init__(self, print( f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}" ) - if apply_chat_template: - assert False + import traceback + traceback.print_stack() if self.MULTIMODAL: apply_chat_template = True logger.info( From 07346264dba89958ea36ad466627b1b4be477c19 Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Tue, 26 Aug 2025 03:27:56 +0000 Subject: [PATCH 17/19] add more debug info Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tests/integration/defs/accuracy/accuracy_core.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index e4fc6be5c3c0..fec615e57d1a 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -189,6 +189,8 @@ def evaluate(self, sampling_params.truncate_prompt_tokens = self.MAX_INPUT_LEN evaluator_kwargs = {} + print(f"self.EVALUATOR_KWARGS: {self.EVALUATOR_KWARGS}") + print(f"extra_evaluator_kwargs: {extra_evaluator_kwargs}") if self.EVALUATOR_KWARGS is not None: evaluator_kwargs.update(self.EVALUATOR_KWARGS) if extra_evaluator_kwargs is not None: From ccaf542d17b8aef8dd93aa2b35e88971bbad177e Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Tue, 26 Aug 2025 07:44:28 +0000 Subject: [PATCH 18/19] fix bugs of TestGPTOSS which affects the member settings of GSM8K Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- .../defs/accuracy/test_llm_api_pytorch.py | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 16ad0852f4a2..921119132286 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2343,11 +2343,9 @@ def test_nvfp4( **pytorch_config, enable_attention_dp=attention_dp, max_batch_size=32) as llm: - # task = MMLU(self.MODEL_NAME) - # task.evaluate(llm) + task = MMLU(self.MODEL_NAME) + task.evaluate(llm) task = GSM8K(self.MODEL_NAME) - task.apply_chat_template = False - print(f"task.apply_chat_template: {task.apply_chat_template}") task.evaluate(llm) assert False @@ -2682,22 +2680,21 @@ def test_auto_dtype_long_rope(self): class TestGPTOSS(LlmapiAccuracyTestHarness): kv_cache_config = KvCacheConfig(free_gpu_memory_fraction=0.5) + extra_evaluator_kwargs = { + "fewshot_as_multiturn": True, + "apply_chat_template": True, + "scores_filter": "exact_match,flexible-extract", + "MAX_OUTPUT_LEN": 8192 + } MODEL_PATH = f"{llm_models_root()}/gpt_oss/gpt-oss-120b" - def update_task_kwargs(self, task): - task.EVALUATOR_KWARGS["fewshot_as_multiturn"] = True - task.EVALUATOR_KWARGS["apply_chat_template"] = True - task.EVALUATE_KWARGS["scores_filter"] = "exact_match,flexible-extract" - task.MAX_OUTPUT_LEN = 8192 - return task - @pytest.mark.parametrize("moe_backend", ["CUTLASS", "TRTLLM", "TRITON"], ids=["cutlass", "trtllm", "triton"]) @pytest.mark.parametrize("cuda_graph,overlap_scheduler", [ (True, True), ]) - def test_w4_1gpu(self, moe_backend, cuda_graph, overlap_scheduler): + def test_w4_1gpu(self, moe_backend, cuda_graph, overlap_scheduler, mocker): if moe_backend == "TRITON" and not IS_TRITON_KERNELS_AVAILABLE: pytest.skip("Triton kernels are not available") @@ -2715,9 +2712,10 @@ def test_w4_1gpu(self, moe_backend, cuda_graph, overlap_scheduler): with llm: model_name = "GPT-OSS/MXFP4" + mocker.patch.object(GSM8K, {"MAX_OUTPUT_LEN": 8192}) task = GSM8K(model_name) - task = self.update_task_kwargs(task) - task.evaluate(llm) + task.evaluate(llm, + extra_evaluator_kwargs=self.extra_evaluator_kwargs) @pytest.mark.skip_less_device(4) @pytest.mark.parametrize("moe_backend", ["CUTLASS", "TRTLLM", "TRITON"]) @@ -2729,7 +2727,7 @@ def test_w4_1gpu(self, moe_backend, cuda_graph, overlap_scheduler): ], ids=["tp4", "ep4", "dp4"]) def test_w4_4gpus(self, moe_backend, tp_size, pp_size, ep_size, - attention_dp, cuda_graph, overlap_scheduler): + attention_dp, cuda_graph, overlap_scheduler, mocker): if moe_backend == "TRITON": if not IS_TRITON_KERNELS_AVAILABLE: pytest.skip("Triton kernels are not available") @@ -2750,8 +2748,10 @@ def test_w4_4gpus(self, moe_backend, tp_size, pp_size, ep_size, with llm: model_name = "GPT-OSS/MXFP4" task = GSM8K(model_name) - task = self.update_task_kwargs(task) - task.evaluate(llm) + + mocker.patch.object(GSM8K, {"MAX_OUTPUT_LEN": 8192}) + task.evaluate(llm, + extra_evaluator_kwargs=self.extra_evaluator_kwargs) @pytest.mark.skip_less_device(4) @pytest.mark.parametrize( @@ -2760,7 +2760,7 @@ def test_w4_4gpus(self, moe_backend, tp_size, pp_size, ep_size, ], ids=["dp4"]) def test_w4a16(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, - overlap_scheduler, monkeypatch): + overlap_scheduler, monkeypatch, mocker): if not IS_TRITON_KERNELS_AVAILABLE: pytest.skip("Triton kernels are not available") monkeypatch.setenv("OVERRIDE_QUANT_ALGO", "W4A16_MXFP4") @@ -2780,8 +2780,9 @@ def test_w4a16(self, tp_size, pp_size, ep_size, attention_dp, cuda_graph, with llm: model_name = "GPT-OSS/BF16" task = GSM8K(model_name) - task = self.update_task_kwargs(task) - task.evaluate(llm) + mocker.patch.object(GSM8K, {"MAX_OUTPUT_LEN": 8192}) + task.evaluate(llm, + extra_evaluator_kwargs=self.extra_evaluator_kwargs) class TestEXAONE4(LlmapiAccuracyTestHarness): From e3d47b13294acfe8208d20210847c9a9a8d35f4e Mon Sep 17 00:00:00 2001 From: bhsueh <11360707+byshiue@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:17:00 +0000 Subject: [PATCH 19/19] revert changes for debugging Signed-off-by: bhsueh <11360707+byshiue@users.noreply.github.com> --- tensorrt_llm/evaluate/lm_eval.py | 13 ------------- tests/integration/defs/accuracy/accuracy_core.py | 8 ++------ .../defs/accuracy/test_llm_api_pytorch.py | 2 -- 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/tensorrt_llm/evaluate/lm_eval.py b/tensorrt_llm/evaluate/lm_eval.py index 84bce6eee441..c77e3be4479e 100644 --- a/tensorrt_llm/evaluate/lm_eval.py +++ b/tensorrt_llm/evaluate/lm_eval.py @@ -311,11 +311,6 @@ def __init__(self, ) from e import lm_eval.tasks self.MULTIMODAL = is_multimodal - print( - f"LmEvalEvaluator::__init__: apply_chat_template: {apply_chat_template}" - ) - import traceback - traceback.print_stack() if self.MULTIMODAL: apply_chat_template = True logger.info( @@ -394,9 +389,6 @@ def evaluate(self, scores_filter: str = None) -> float: import lm_eval lm_cls = MultimodalLmEvalWrapper if self.MULTIMODAL else LmEvalWrapper - print( - f"LmEvalEvaluator::evaluate: apply_chat_template: {self.apply_chat_template}" - ) results = lm_eval.evaluate( lm=lm_cls(llm, sampling_params, streaming), task_dict=self.task_dict, @@ -406,11 +398,6 @@ def evaluate(self, system_instruction=self.system_prompt) # Normalize scores to range 0~100 scores = results["results"][self.task_name] - if self.task_name == "gsm8k": - print(f"scores: {scores}, results: {results}") - print(f"scores_filter: {scores_filter}") - import sys - sys.stdout.flush() for metric in scores.keys(): if isinstance(scores[metric], (float, int)): scores[metric] *= 100 diff --git a/tests/integration/defs/accuracy/accuracy_core.py b/tests/integration/defs/accuracy/accuracy_core.py index fec615e57d1a..06303886292b 100644 --- a/tests/integration/defs/accuracy/accuracy_core.py +++ b/tests/integration/defs/accuracy/accuracy_core.py @@ -189,8 +189,6 @@ def evaluate(self, sampling_params.truncate_prompt_tokens = self.MAX_INPUT_LEN evaluator_kwargs = {} - print(f"self.EVALUATOR_KWARGS: {self.EVALUATOR_KWARGS}") - print(f"extra_evaluator_kwargs: {extra_evaluator_kwargs}") if self.EVALUATOR_KWARGS is not None: evaluator_kwargs.update(self.EVALUATOR_KWARGS) if extra_evaluator_kwargs is not None: @@ -299,15 +297,13 @@ class GSM8K(AccuracyTask): ALPHA = 0.05 BETA = 0.2 SIGMA = 50 - NUM_SAMPLES = 2 # Full sample + NUM_SAMPLES = 1319 # Full sample MAX_INPUT_LEN = 4096 MAX_OUTPUT_LEN = 256 EVALUATOR_CLS = tensorrt_llm.evaluate.GSM8K - EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, - random_seed=0, - apply_chat_template=False) + EVALUATOR_KWARGS = dict(dataset_path=DATASET_DIR, random_seed=0) EVALUATE_KWARGS = dict(scores_filter=None) diff --git a/tests/integration/defs/accuracy/test_llm_api_pytorch.py b/tests/integration/defs/accuracy/test_llm_api_pytorch.py index 921119132286..223c97aa58b7 100644 --- a/tests/integration/defs/accuracy/test_llm_api_pytorch.py +++ b/tests/integration/defs/accuracy/test_llm_api_pytorch.py @@ -2347,7 +2347,6 @@ def test_nvfp4( task.evaluate(llm) task = GSM8K(self.MODEL_NAME) task.evaluate(llm) - assert False def test_eagle3(self): pytorch_config = dict( @@ -2748,7 +2747,6 @@ def test_w4_4gpus(self, moe_backend, tp_size, pp_size, ep_size, with llm: model_name = "GPT-OSS/MXFP4" task = GSM8K(model_name) - mocker.patch.object(GSM8K, {"MAX_OUTPUT_LEN": 8192}) task.evaluate(llm, extra_evaluator_kwargs=self.extra_evaluator_kwargs)