diff --git a/src/diffusers/pipelines/kolors/tokenizer.py b/src/diffusers/pipelines/kolors/tokenizer.py index 5fc475ddc079..50efe7aefa5f 100644 --- a/src/diffusers/pipelines/kolors/tokenizer.py +++ b/src/diffusers/pipelines/kolors/tokenizer.py @@ -137,6 +137,9 @@ def __init__( padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, encode_special_tokens=encode_special_tokens, + pad_token=kwargs.pop("pad_token", ""), + unk_token=kwargs.pop("unk_token", ""), + eos_token=kwargs.pop("eos_token", ""), **kwargs, ) @@ -148,19 +151,19 @@ def get_command(self, token): @property def unk_token(self) -> str: - return "" + return self._special_tokens_map.get("unk_token", "") @unk_token.setter def unk_token(self, value: str): - self._unk_token = value + self._special_tokens_map["unk_token"] = value @property def pad_token(self) -> str: - return "" + return self._special_tokens_map.get("pad_token", "") @pad_token.setter def pad_token(self, value: str): - self._pad_token = value + self._special_tokens_map["pad_token"] = value @property def pad_token_id(self): @@ -168,11 +171,11 @@ def pad_token_id(self): @property def eos_token(self) -> str: - return "" + return self._special_tokens_map.get("eos_token", "") @eos_token.setter def eos_token(self, value: str): - self._eos_token = value + self._special_tokens_map["eos_token"] = value @property def eos_token_id(self): diff --git a/tests/pipelines/kolors/test_kolors.py b/tests/pipelines/kolors/test_kolors.py index adfca799de18..3b3e6fef1528 100644 --- a/tests/pipelines/kolors/test_kolors.py +++ b/tests/pipelines/kolors/test_kolors.py @@ -145,3 +145,18 @@ def test_save_load_float16(self): def test_inference_batch_single_identical(self): self._test_inference_batch_single_identical(expected_max_diff=5e-3) + + def test_chatglm_tokenizer_properties(self): + components = self.get_dummy_components() + tokenizer = components["tokenizer"] + self.assertEqual(tokenizer.pad_token, "") + self.assertEqual(tokenizer.unk_token, "") + self.assertEqual(tokenizer.eos_token, "") + + tokenizer.pad_token = "" + tokenizer.unk_token = "" + tokenizer.eos_token = "" + + self.assertEqual(tokenizer.pad_token, "") + self.assertEqual(tokenizer.unk_token, "") + self.assertEqual(tokenizer.eos_token, "")