Skip to content

Allow tokenizers 0.23.1 #46381

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ArthurZucker wants to merge 11 commits into
base: main
Choose a base branch
from
Open

Allow tokenizers 0.23.1 #46381

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
cead6ed
Allow tokenizers 0.23.1
ArthurZucker Jun 2, 2026
876dc36
Fix RobertaProcessing kwargs for tokenizers 0.23.1
ArthurZucker Jun 3, 2026
6006a38
Tolerate dropped overflow segment in max-encoding-length tests
ArthurZucker Jun 3, 2026
9632dfe
Drop overflow-segment assertions in max-encoding-length tests
ArthurZucker Jun 3, 2026
9b77e50
Apply suggestion from @ArthurZucker
ArthurZucker Jun 4, 2026
e764f2c
Fix HerbertTokenizer: pass BertProcessing sep/cls positionally
ArthurZucker Jun 9, 2026
61e16c3
Update dependency versions table for tokenizers 0.23.1 pin
ArthurZucker Jun 9, 2026
af95ba9
Remove accidentally committed test_opt tokenizer artifacts
ArthurZucker Jun 9, 2026
6c8ef7a
Fix ty type errors in tokenization files
ArthurZucker Jun 9, 2026
f613d6e
Merge branch 'main' into bump-tokenizers-0.23.1
ArthurZucker Jul 17, 2026
bbb6c47
Pass BertProcessing/RobertaProcessing args positionally in call sites…
ArthurZucker Jul 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@
"tomli",
"tiktoken",
"timm>=1.0.23",
"tokenizers>=0.22.0,<=0.23.0",
"tokenizers>=0.23.1,<0.24.0",
"torch>=2.4",
"torchaudio",
"torchvision",
Expand Down
20 changes: 10 additions & 10 deletions src/transformers/convert_slow_tokenizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -481,8 +481,8 @@ def converted(self) -> Tokenizer:
tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
tokenizer.decoder = decoders.BPEDecoder(suffix=token_suffix)
tokenizer.post_processor = processors.BertProcessing(
sep=(self.original_tokenizer.sep_token, self.original_tokenizer.sep_token_id),
cls=(self.original_tokenizer.cls_token, self.original_tokenizer.cls_token_id),
(self.original_tokenizer.sep_token, self.original_tokenizer.sep_token_id),
(self.original_tokenizer.cls_token, self.original_tokenizer.cls_token_id),
)

return tokenizer
Expand Down Expand Up @@ -552,10 +552,10 @@ def converted(self) -> Tokenizer:
tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=ot.add_prefix_space)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.RobertaProcessing(
sep=(ot.sep_token, ot.sep_token_id),
cls=(ot.cls_token, ot.cls_token_id),
add_prefix_space=ot.add_prefix_space,
trim_offsets=True, # True by default on Roberta (historical)
(ot.sep_token, ot.sep_token_id),
(ot.cls_token, ot.cls_token_id),
True, # trim_offsets: True by default on Roberta (historical)
ot.add_prefix_space,
)

return tokenizer
Expand Down Expand Up @@ -1459,10 +1459,10 @@ def converted(self) -> Tokenizer:

# Hack to have a ByteLevel and TemplateProcessor
tokenizer.post_processor = processors.RobertaProcessing(
sep=(self.original_tokenizer.eos_token, self.original_tokenizer.eos_token_id),
cls=(self.original_tokenizer.bos_token, self.original_tokenizer.bos_token_id),
add_prefix_space=False,
trim_offsets=False,
(self.original_tokenizer.eos_token, self.original_tokenizer.eos_token_id),
(self.original_tokenizer.bos_token, self.original_tokenizer.bos_token_id),
False, # trim_offsets
False, # add_prefix_space
)
return tokenizer

Expand Down
2 changes: 1 addition & 1 deletion src/transformers/dependency_versions_table.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
"tomli": "tomli",
"tiktoken": "tiktoken",
"timm": "timm>=1.0.23",
"tokenizers": "tokenizers>=0.22.0,<=0.23.0",
"tokenizers": "tokenizers>=0.23.1,<0.24.0",
"torch": "torch>=2.4",
"torchaudio": "torchaudio",
"torchvision": "torchvision",
Expand Down
1 change: 1 addition & 0 deletions src/transformers/models/bertweet/tokenization_bertweet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
"""Tokenization classes for BERTweet"""

import html
import html.entities
import os
import re

Expand Down
8 changes: 4 additions & 4 deletions src/transformers/models/clip/tokenization_clip.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,10 @@ def __init__(
)

self._tokenizer.post_processor = processors.RobertaProcessing(
sep=(str(eos_token), self.eos_token_id),
cls=(str(bos_token), self.bos_token_id),
add_prefix_space=False,
trim_offsets=False,
(str(eos_token), self.eos_token_id),
(str(bos_token), self.bos_token_id),
False, # trim_offsets
False, # add_prefix_space
)

# Very ugly hack to enable padding to have a correct decoding see https://github.com/huggingface/tokenizers/issues/872
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/herbert/tokenization_herbert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,8 @@ def __init__(
)

self._tokenizer.post_processor = processors.BertProcessing(
sep=(self.sep_token, 2),
cls=(self.cls_token, 0),
(self.sep_token, 2),
(self.cls_token, 0),
)


Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/layoutlmv2/tokenization_layoutlmv2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ def __call__(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -401,7 +401,7 @@ def batch_encode_plus(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -466,7 +466,7 @@ def encode_plus(
word_labels: list[int] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down
14 changes: 7 additions & 7 deletions src/transformers/models/layoutlmv3/tokenization_layoutlmv3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,10 +226,10 @@ def __init__(
sep_token_id = self.sep_token_id

self._tokenizer.post_processor = processors.RobertaProcessing(
sep=(sep, sep_token_id),
cls=(cls, cls_token_id),
add_prefix_space=add_prefix_space,
trim_offsets=True,
(sep, sep_token_id),
(cls, cls_token_id),
True, # trim_offsets
add_prefix_space,
)
self.cls_token_box = cls_token_box
self.sep_token_box = sep_token_box
Expand All @@ -246,7 +246,7 @@ def __call__(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -398,7 +398,7 @@ def batch_encode_plus(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -463,7 +463,7 @@ def encode_plus(
word_labels: list[int] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/layoutxlm/tokenization_layoutxlm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ def encode_plus(
word_labels: list[int] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -359,7 +359,7 @@ def batch_encode_plus(
word_labels: list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -419,7 +419,7 @@ def __call__(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/luke/tokenization_luke.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ def __call__(
entities_pair: EntityInput | list[EntityInput] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
max_entity_length: int | None = None,
stride: int = 0,
Expand Down Expand Up @@ -1133,7 +1133,7 @@ def prepare_for_model(
pair_entity_token_spans: list[tuple[int, int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
max_entity_length: int | None = None,
stride: int = 0,
Expand Down
6 changes: 3 additions & 3 deletions src/transformers/models/markuplm/tokenization_markuplm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ def __call__(
node_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
Expand Down Expand Up @@ -459,7 +459,7 @@ def batch_encode_plus(
node_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -524,7 +524,7 @@ def encode_plus(
node_labels: list[int] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/mluke/tokenization_mluke.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@ def __call__(
entities_pair: EntityInput | list[EntityInput] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
max_entity_length: int | None = None,
stride: int = 0,
Expand Down Expand Up @@ -1171,7 +1171,7 @@ def prepare_for_model(
pair_entity_token_spans: list[tuple[int, int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
max_entity_length: int | None = None,
stride: int = 0,
Expand Down
8 changes: 4 additions & 4 deletions src/transformers/models/mpnet/tokenization_mpnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,10 @@ def __init__(
sep_token_id = self.sep_token_id if self.sep_token_id is not None else 2

self._tokenizer.post_processor = processors.RobertaProcessing(
sep=(sep_str, sep_token_id),
cls=(cls_str, cls_token_id),
trim_offsets=True,
add_prefix_space=False,
(sep_str, sep_token_id),
(cls_str, cls_token_id),
True, # trim_offsets
False, # add_prefix_space
)

@property
Expand Down
4 changes: 2 additions & 2 deletions src/transformers/models/pop2piano/tokenization_pop2piano.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,7 @@ def relative_tokens_ids_to_notes(self, tokens: np.ndarray, start_idx: float, cut
notes = notes[note_order.argsort()]
return notes

def notes_to_midi(self, notes: np.ndarray, beatstep: np.ndarray, offset_sec: int = 0.0):
def notes_to_midi(self, notes: np.ndarray, beatstep: np.ndarray, offset_sec: float = 0.0):
"""
Converts notes to Midi.

Expand Down Expand Up @@ -474,7 +474,7 @@ def __call__(
self,
notes: np.ndarray | list[pretty_midi.Note] | list[list[pretty_midi.Note]],
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
pad_to_multiple_of: int | None = None,
return_attention_mask: bool | None = None,
Expand Down
8 changes: 4 additions & 4 deletions src/transformers/models/roberta/tokenization_roberta.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,10 +168,10 @@ def __init__(
**kwargs,
)
self._tokenizer.post_processor = processors.RobertaProcessing(
sep=(str(sep_token), self.sep_token_id),
cls=(str(cls_token), self.cls_token_id),
add_prefix_space=add_prefix_space,
trim_offsets=trim_offsets,
(str(sep_token), self.sep_token_id),
(str(cls_token), self.cls_token_id),
trim_offsets,
add_prefix_space,
)


Expand Down
8 changes: 4 additions & 4 deletions src/transformers/models/roc_bert/tokenization_roc_bert.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def __call__(
text_target: TextInput | list[TextInput] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
max_target_length: int | None = None,
stride: int = 0,
Expand Down Expand Up @@ -299,7 +299,7 @@ def encode_plus(
text_pair: TextInput | PreTokenizedInput | EncodedInput | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
Expand Down Expand Up @@ -356,7 +356,7 @@ def batch_encode_plus(
| list[EncodedInputPair],
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
Expand Down Expand Up @@ -535,7 +535,7 @@ def prepare_for_model(
pair_pronunciation_ids: list[int] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down
2 changes: 1 addition & 1 deletion src/transformers/models/tapas/tokenization_tapas.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2539,7 +2539,7 @@ def parse_text(text):


_PrimitiveNumericValue = float | tuple[float | None]
_SortKeyFn = Callable[[NumericValue], tuple[float, Ellipsis]]
_SortKeyFn = Callable[[NumericValue], tuple[float, ...]]

_DATE_TUPLE_SIZE = 3

Expand Down
12 changes: 6 additions & 6 deletions src/transformers/models/udop/tokenization_udop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,11 @@ def __init__(
@add_end_docstrings(UDOP_ENCODE_KWARGS_DOCSTRING)
def __call__(
self,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
text_pair: PreTokenizedInput | list[PreTokenizedInput] | None = None,
boxes: list[list[int]] | list[list[list[int]]] | None = None,
word_labels: list[int] | list[list[int]] | None = None,
text_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] = None,
text_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
text_pair_target: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput] | None = None,
**kwargs,
) -> BatchEncoding:
Expand Down Expand Up @@ -305,7 +305,7 @@ def call_boxes(
word_labels: list[int] | list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
pad_to_multiple_of: int | None = None,
Expand Down Expand Up @@ -467,7 +467,7 @@ def batch_encode_plus_boxes(
word_labels: list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
Expand Down Expand Up @@ -767,7 +767,7 @@ def encode_boxes(
word_labels: list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
return_tensors: str | TensorType | None = None,
Expand Down Expand Up @@ -810,7 +810,7 @@ def encode_plus_boxes(
word_labels: list[list[int]] | None = None,
add_special_tokens: bool = True,
padding: bool | str | PaddingStrategy = False,
truncation: bool | str | TruncationStrategy = None,
truncation: bool | str | TruncationStrategy | None = None,
max_length: int | None = None,
stride: int = 0,
is_split_into_words: bool = False,
Expand Down
Loading
Loading