load PreTrainedSentencePieceTokenizer fallback

Author: itazap

src/transformers/__init__.py

@@ -175,6 +175,7 @@
     "processing_utils": ["ProcessorMixin"],
     "quantizers": [],
     "testing_utils": [],
+    "tokenization_sentencepiece": ["PreTrainedSentencePieceTokenizer"],
     "tokenization_utils": ["PreTrainedTokenizer"],
     "tokenization_utils_base": [
         "AddedToken",
@@ -710,6 +711,7 @@
     from .pytorch_utils import prune_layer as prune_layer
 
     # Tokenization
+    from .tokenization_sentencepiece import PreTrainedSentencePieceTokenizer as PreTrainedSentencePieceTokenizer
     from .tokenization_utils import PreTrainedTokenizer as PreTrainedTokenizer
     from .tokenization_utils_base import AddedToken as AddedToken
     from .tokenization_utils_base import BatchEncoding as BatchEncoding

src/transformers/models/auto/tokenization_auto.py

@@ -26,11 +26,13 @@
 from ...configuration_utils import PretrainedConfig
 from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
 from ...modeling_gguf_pytorch_utils import load_gguf_checkpoint
+from ...tokenization_sentencepiece import PreTrainedSentencePieceTokenizer
 from ...tokenization_utils import PreTrainedTokenizer
 from ...tokenization_utils_base import TOKENIZER_CONFIG_FILE
 from ...utils import (
     cached_file,
     extract_commit_hash,
+    has_file,
     is_g2p_en_available,
     is_sentencepiece_available,
     is_tokenizers_available,
@@ -356,7 +358,7 @@
         (
             "llama",
             (
-                "LlamaTokenizer" if is_sentencepiece_available() else None,
+                None,
                 "LlamaTokenizerFast" if is_tokenizers_available() else None,
             ),
         ),
@@ -1133,6 +1135,31 @@ def from_pretrained(cls, pretrained_model_name_or_path, *inputs, **kwargs):
                 tokenizer_class_candidate = config_tokenizer_class
                 tokenizer_class = tokenizer_class_from_name(tokenizer_class_candidate)
             if tokenizer_class is None:
+                try:
+                    vocab_file_exists = has_file(
+                        pretrained_model_name_or_path,
+                        "tokenizer.model",
+                        revision=kwargs.get("revision", None),
+                        token=kwargs.get("token", None),
+                        cache_dir=kwargs.get("cache_dir", None),
+                        local_files_only=kwargs.get("local_files_only", False),
+                    )
+                except Exception:
+                    vocab_file_exists = False
+                
+                if vocab_file_exists:
+                    logger.info(
+                        "Falling back to PreTrainedSentencePieceTokenizer since tokenizer.model file was found "
+                        "but no config or tokenizer class could be determined."
+                    )
+                    return PreTrainedSentencePieceTokenizer.from_pretrained(
+                        pretrained_model_name_or_path, *inputs, **kwargs
+                    )
+                
+                raise ValueError(
+                    f"Could not load tokenizer from {pretrained_model_name_or_path}. "
+                    "No tokenizer configuration or model config could be found."
+                )
                 raise ValueError(
                     f"Tokenizer class {tokenizer_class_candidate} does not exist or is not currently imported."
                 )