rm slow tokenizers

src/transformers/__init__.py

@@ -175,6 +175,7 @@
     "processing_utils": ["ProcessorMixin"],
     "quantizers": [],
     "testing_utils": [],
+    "tokenization_sentencepiece": ["SentencePieceBackend"],
     "tokenization_utils": ["PreTrainedTokenizer"],
     "tokenization_utils_base": [
         "AddedToken",
@@ -280,7 +281,7 @@
     ]
 else:
     # Fast tokenizers structure
-    _import_structure["tokenization_utils_fast"] = ["PreTrainedTokenizerFast"]
+    _import_structure["tokenization_tokenizers"] = ["TokenizersBackend", "PreTrainedTokenizerFast"]
 
 
 try:
@@ -710,14 +711,15 @@
     from .pytorch_utils import prune_layer as prune_layer
 
     # Tokenization
+    from .tokenization_sentencepiece import SentencePieceBackend as SentencePieceBackend
     from .tokenization_utils import PreTrainedTokenizer as PreTrainedTokenizer
     from .tokenization_utils_base import AddedToken as AddedToken
     from .tokenization_utils_base import BatchEncoding as BatchEncoding
     from .tokenization_utils_base import CharSpan as CharSpan
     from .tokenization_utils_base import PreTrainedTokenizerBase as PreTrainedTokenizerBase
-    from .tokenization_utils_base import SpecialTokensMixin as SpecialTokensMixin
     from .tokenization_utils_base import TokenSpan as TokenSpan
-    from .tokenization_utils_fast import PreTrainedTokenizerFast as PreTrainedTokenizerFast
+    from .tokenization_tokenizers import TokenizersBackend as TokenizersBackend
+    from .tokenization_tokenizers import PreTrainedTokenizerFast as PreTrainedTokenizerFast
 
     # Trainer
     from .trainer import Trainer as Trainer

src/transformers/data/processors/squad.py

@@ -162,7 +162,8 @@ def squad_convert_example_to_features(
         if tokenizer_type in MULTI_SEP_TOKENS_TOKENIZERS_SET
         else tokenizer.model_max_length - tokenizer.max_len_single_sentence
     )
-    sequence_pair_added_tokens = tokenizer.model_max_length - tokenizer.max_len_sentences_pair
+    max_len_sentences_pair = self.model_max_length - self.num_special_tokens_to_add(pair=True)
+    sequence_pair_added_tokens = tokenizer.model_max_length
 
     span_doc_tokens = all_doc_tokens
     while len(spans) * doc_stride < len(all_doc_tokens):

src/transformers/models/auto/tokenization_auto.py

@@ -15,6 +15,7 @@
 """Auto Tokenizer class."""
 
 import importlib
+import inspect
 import json
 import os
 import warnings
@@ -28,6 +29,7 @@
 from ...modeling_gguf_pytorch_utils import load_gguf_checkpoint
 from ...tokenization_utils import PreTrainedTokenizer
 from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
+from ...utils.hub import has_file, cached_file
 from ...utils import (
     cached_file,
     extract_commit_hash,
@@ -46,12 +48,10 @@
     replace_list_option_in_docstrings,
 )
 
-
 if is_tokenizers_available():
-    from ...tokenization_utils_fast import PreTrainedTokenizerFast
+    from ...tokenization_tokenizers import TokenizersBackend
 else:
-    PreTrainedTokenizerFast = None
-
+    TokenizersBackend = None
 
 logger = logging.get_logger(__name__)
 
@@ -357,7 +357,7 @@
             "llama",
             (
                 "LlamaTokenizer" if is_sentencepiece_available() else None,
-                "LlamaTokenizerFast" if is_tokenizers_available() else None,
+                "LlamaTokenizer" if is_tokenizers_available() else None,
             ),
         ),
         (
@@ -654,7 +654,7 @@
         ("speech_to_text", ("Speech2TextTokenizer" if is_sentencepiece_available() else None, None)),
         ("speech_to_text_2", ("Speech2Text2Tokenizer", None)),
         ("speecht5", ("SpeechT5Tokenizer" if is_sentencepiece_available() else None, None)),
-        ("splinter", ("SplinterTokenizer", "SplinterTokenizerFast")),
+        ("splinter", ("SplinterTokenizer" if is_tokenizers_available() else None, None)),
         (
             "squeezebert",
             ("SqueezeBertTokenizer", "SqueezeBertTokenizerFast" if is_tokenizers_available() else None),
@@ -720,13 +720,7 @@
         ("wav2vec2_phoneme", ("Wav2Vec2PhonemeCTCTokenizer", None)),
         ("whisper", ("WhisperTokenizer", "WhisperTokenizerFast" if is_tokenizers_available() else None)),
         ("xclip", ("CLIPTokenizer", "CLIPTokenizerFast" if is_tokenizers_available() else None)),
-        (
-            "xglm",
-            (
-                "XGLMTokenizer" if is_sentencepiece_available() else None,
-                "XGLMTokenizerFast" if is_tokenizers_available() else None,
-            ),
-        ),
+        ("xglm", ("XGLMTokenizer" if is_tokenizers_available() else None, None)),
         ("xlm", ("XLMTokenizer", None)),
         ("xlm-prophetnet", ("XLMProphetNetTokenizer" if is_sentencepiece_available() else None, None)),
         (
@@ -818,16 +812,16 @@ def tokenizer_class_from_name(class_name: str) -> Union[type[Any], None]:
 
 
 def get_tokenizer_config(
-    pretrained_model_name_or_path: Union[str, os.PathLike[str]],
-    cache_dir: Optional[Union[str, os.PathLike[str]]] = None,
-    force_download: bool = False,
-    resume_download: Optional[bool] = None,
-    proxies: Optional[dict[str, str]] = None,
-    token: Optional[Union[bool, str]] = None,
-    revision: Optional[str] = None,
-    local_files_only: bool = False,
-    subfolder: str = "",
-    **kwargs,
+        pretrained_model_name_or_path: Union[str, os.PathLike[str]],
+        cache_dir: Optional[Union[str, os.PathLike[str]]] = None,
+        force_download: bool = False,
+        resume_download: Optional[bool] = None,
+        proxies: Optional[dict[str, str]] = None,
+        token: Optional[Union[bool, str]] = None,
+        revision: Optional[str] = None,
+        local_files_only: bool = False,
+        subfolder: str = "",
+        **kwargs,
 ) -> dict[str, Any]:
     """
     Loads the tokenizer configuration from a pretrained model tokenizer configuration.
@@ -1098,11 +1092,11 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
 
         has_remote_code = tokenizer_auto_map is not None
         has_local_code = type(config) in TOKENIZER_MAPPING or (
-            config_tokenizer_class is not None
-            and (
-                tokenizer_class_from_name(config_tokenizer_class) is not None
-                or tokenizer_class_from_name(config_tokenizer_class + "Fast") is not None
-            )
+                config_tokenizer_class is not None
+                and (
+                        tokenizer_class_from_name(config_tokenizer_class) is not None
+                        or tokenizer_class_from_name(config_tokenizer_class + "Fast") is not None
+                )
         )
         if has_remote_code:
             if use_fast and tokenizer_auto_map[1] is not None:
@@ -1126,15 +1120,102 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
             )
         elif config_tokenizer_class is not None:
             tokenizer_class = None
-            if use_fast and not config_tokenizer_class.endswith("Fast"):
-                tokenizer_class_candidate = f"{config_tokenizer_class}Fast"
-                tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
-            if tokenizer_class is None:
+            fast_tokenizer_class = None
+            if use_fast:
+                fast_candidate = (
+                    f"{config_tokenizer_class}Fast" if not config_tokenizer_class.endswith("Fast") else config_tokenizer_class
+                )
+                fast_tokenizer_class = tokenizer_class_from_name(fast_candidate)
+            if fast_tokenizer_class is None:
                 tokenizer_class_candidate = config_tokenizer_class
                 tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
+            else:
+                tokenizer_class_candidate = fast_candidate
+                tokenizer_class = fast_tokenizer_class
+
+            if use_fast and fast_tokenizer_class is not None:
+                # If tokenizer.json exists, load the tokenizers-backend 
+                try:
+                    tokenizer_json_exists = has_file(
+                        pretrained_model_name_or_path,
+                        "tokenizer.json",
+                        revision=kwargs.get("revision", None),
+                        token=kwargs.get("token", None),
+                        cache_dir=kwargs.get("cache_dir", None),
+                        local_files_only=kwargs.get("local_files_only", False),
+                    )
+                except Exception:
+                    tokenizer_json_exists = False
+                if tokenizer_json_exists:
+                    return fast_tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
+
+                # If only SPM exists, try to get vocab and merges and init to load a tokenizers-backend
+                try:
+                    spm_exists = has_file(
+                        pretrained_model_name_or_path,
+                        "tokenizer.model",
+                        revision=kwargs.get("revision", None),
+                        token=kwargs.get("token", None),
+                        cache_dir=kwargs.get("cache_dir", None),
+                        local_files_only=kwargs.get("local_files_only", False),
+                    )
+                except Exception:
+                    spm_exists = False
+                if spm_exists:
+                    try:
+                        resolved_spm = cached_file(
+                            pretrained_model_name_or_path,
+                            "tokenizer.model",
+                            cache_dir=kwargs.get("cache_dir", None),
+                            force_download=kwargs.get("force_download", False),
+                            proxies=kwargs.get("proxies", None),
+                            token=kwargs.get("token", None),
+                            revision=kwargs.get("revision", None),
+                            local_files_only=kwargs.get("local_files_only", False),
+                            subfolder=kwargs.get("subfolder", ""),
+                        )
+                    except Exception:
+                        resolved_spm = None
+                    if resolved_spm is not None:
+                        try:
+                            from ...create_fast_tokenizer import SentencePieceExtractor
+                            fast_sig = inspect.signature(getattr(fast_tokenizer_class, "__init__", fast_tokenizer_class))
+                            if "vocab" in fast_sig.parameters and "merges" in fast_sig.parameters:
+                                try:
+                                    vocab, merges = SentencePieceExtractor(resolved_spm).extract()
+                                    return fast_tokenizer_class.from_pretrained(
+                                        pretrained_model_name_or_path, *inputs, vocab=vocab, merges=merges, **kwargs
+                                    )
+                                except Exception:
+                                    pass
+                        except Exception:
+                            pass
+            # If we failed with a tokenizers-backend, we load from another backend (below only SPM but will add others)
             if tokenizer_class is None:
+                try:
+                    vocab_file_exists = has_file(
+                        pretrained_model_name_or_path,
+                        "tokenizer.model",
+                        revision=kwargs.get("revision", None),
+                        token=kwargs.get("token", None),
+                        cache_dir=kwargs.get("cache_dir", None),
+                        local_files_only=kwargs.get("local_files_only", False),
+                    )
+                except Exception:
+                    vocab_file_exists = False
+
+                if vocab_file_exists:
+                    logger.info(
+                        "Falling back to SentencePieceBackend since tokenizer.model file was found "
+                        "but no config or tokenizer class could be determined."
+                    )
+                    return SentencePieceBackend.from_pretrained(
+                        pretrained_model_name_or_path, *inputs, **kwargs
+                    )
+
                 raise ValueError(
-                    f"Tokenizer class {tokenizer_class_candidate} does not exist or is not currently imported."
+                    f"Could not load tokenizer from {pretrained_model_name_or_path}. "
+                    "No tokenizer configuration or model config could be found."
                 )
             return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
 

src/transformers/models/cohere/tokenization_cohere_fast.py

@@ -160,16 +160,7 @@ def __init__(
 
         self.add_prefix_space = add_prefix_space
 
-    def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding:
-        is_split_into_words = kwargs.get("is_split_into_words", False)
-        if not (self.add_prefix_space or not is_split_into_words):
-            raise Exception(
-                f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
-                " pretokenized inputs."
-            )
-
-        return super()._batch_encode_plus(*args, **kwargs)
-
+    #TODO: does this check need to be model-specific?
     def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
         is_split_into_words = kwargs.get("is_split_into_words", False)
 
@@ -181,50 +172,6 @@ def _encode_plus(self, *args, **kwargs) -> BatchEncoding:
 
         return super()._encode_plus(*args, **kwargs)
 
-    def update_post_processor(self):
-        """
-        Updates the underlying post processor with the current `bos_token` and `eos_token`.
-        """
-        bos = self.bos_token
-        bos_token_id = self.bos_token_id
-        if bos is None and self.add_bos_token:
-            raise ValueError("add_bos_token = True but bos_token = None")
-
-        eos = self.eos_token
-        eos_token_id = self.eos_token_id
-        if eos is None and self.add_eos_token:
-            raise ValueError("add_eos_token = True but eos_token = None")
-
-        single = f"{(bos + ':0 ') if self.add_bos_token else ''}$A:0{(' ' + eos + ':0') if self.add_eos_token else ''}"
-        pair = f"{single}{(' ' + bos + ':1') if self.add_bos_token else ''} $B:1{(' ' + eos + ':1') if self.add_eos_token else ''}"
-
-        special_tokens = []
-        if self.add_bos_token:
-            special_tokens.append((bos, bos_token_id))
-        if self.add_eos_token:
-            special_tokens.append((eos, eos_token_id))
-        self._tokenizer.post_processor = processors.TemplateProcessing(
-            single=single, pair=pair, special_tokens=special_tokens
-        )
-
-    @property
-    def add_eos_token(self):
-        return self._add_eos_token
-
-    @property
-    def add_bos_token(self):
-        return self._add_bos_token
-
-    @add_eos_token.setter
-    def add_eos_token(self, value):
-        self._add_eos_token = value
-        self.update_post_processor()
-
-    @add_bos_token.setter
-    def add_bos_token(self, value):
-        self._add_bos_token = value
-        self.update_post_processor()
-
     def apply_tool_use_template(
         self,
         conversation: list[dict[str, str]],
@@ -492,17 +439,5 @@ def apply_grounded_generation_template(
             **kwargs,
         )
 
-    # TODO ArthurZ let's rely on the template processor instead, refactor all fast tokenizers
-    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
-        bos_token_id = [self.bos_token_id] if self.add_bos_token else []
-        eos_token_id = [self.eos_token_id] if self.add_eos_token else []
-
-        output = bos_token_id + token_ids_0 + eos_token_id
-
-        if token_ids_1 is not None:
-            output = output + bos_token_id + token_ids_1 + eos_token_id
-
-        return output
-
 
 __all__ = ["CohereTokenizerFast"]