diff --git a/tensorrt_llm/tokenizer/tokenizer.py b/tensorrt_llm/tokenizer/tokenizer.py index 9715865c65a1..b88b6b53ea45 100644 --- a/tensorrt_llm/tokenizer/tokenizer.py +++ b/tensorrt_llm/tokenizer/tokenizer.py @@ -65,6 +65,12 @@ class TokenizerBase(PreTrainedTokenizerBase): ''' This is a protocol for the tokenizer. Users can implement their own tokenizer by inheriting this class. ''' + def __repr__(self) -> str: + # PreTrainedTokenizerBase.__repr__ reads properties (e.g. + # added_tokens_decoder) that TokenizerBase subclasses are not + # required to implement, so fall back to a class-name-only repr. + return f"{self.__class__.__name__}()" + def _reconstruct_transformers_tokenizer(inner_bytes: bytes): '''Reconstruct a TransformersTokenizer from cloudpickle-serialized bytes.''' diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 5fc9451877e0..38a08cdf149f 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -400,7 +400,6 @@ unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) unittest/llmapi/test_additional_model_outputs.py -m "gpu2" SKIP (https://nvbugs/6428091) unittest/llmapi/test_additional_model_outputs.py::test_additional_model_outputs_integration_pp2 SKIP (https://nvbugs/6427411) -unittest/llmapi/test_llm.py::test_llm_with_customized_tokenizer SKIP (https://nvbugs/6507080) unittest/llmapi/test_llm.py::test_tokenizer_decode_incrementally[/scratch.trt_llm_data/llm-models/falcon-7b-instruct-False-0.95-HF] SKIP (https://nvbugs/6507082) unittest/llmapi/test_llm.py::test_tokenizer_decode_incrementally[/scratch.trt_llm_data/llm-models/falcon-7b-instruct-False-0.95-TRTLLM] SKIP (https://nvbugs/6507082) unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092)