rm protobuf dependency

Author: itazap

src/transformers/convert_slow_tokenizer.py

@@ -1334,7 +1334,6 @@ class SpmTokenizer:
 
     def __init__(
         self,
-        vocab_file: str,
         handle_byte_fallback: bool = True,
         legacy: bool = False,
         add_prefix_space: bool = True,
@@ -1346,9 +1345,6 @@ def __init__(
         decoder: Optional[callable] = None,
         post_processor: Optional[callable] = None,
     ):
-        requires_backends(self, "protobuf")
-        
-        self.vocab_file = vocab_file
         self.handle_byte_fallback = handle_byte_fallback
         self.legacy = legacy
         self.add_prefix_space = add_prefix_space
@@ -1360,82 +1356,31 @@ def __init__(
         self._pre_tokenizer_fn = pre_tokenizer
         self._decoder_fn = decoder
         self._post_processor_fn = post_processor
-        
-        # Load the protobuf model
-        model_pb2 = import_protobuf()
-        m = model_pb2.ModelProto()
-        with open(vocab_file, "rb") as f:
-            m.ParseFromString(f.read())
-        self.proto = m
 
-    def vocab(self, proto):
+    def vocab(self):
         if self._vocab_fn is not None:
-            return self._vocab_fn(proto)
-        return [(piece.piece, piece.score) for piece in proto.pieces]
+            return self._vocab_fn()
+        # Return empty vocab for training
+        return []
 
-    def unk_id(self, proto):
+    def unk_id(self):
         if self._unk_id_fn is not None:
-            return self._unk_id_fn(proto)
-        return proto.trainer_spec.unk_id
-
-    def tokenizer(self, proto):
-        model_type = proto.trainer_spec.model_type
-        vocab_scores = self.vocab(proto)
-
-        if model_type == 1:
-            tokenizer = Tokenizer(
-                Unigram(
-                    vocab_scores,
-                    unk_id=self.unk_id(proto),
-                    byte_fallback=self.handle_byte_fallback,
-                )
-            )
-        elif model_type == 2:
-            _, merges = SentencePieceExtractor(self.vocab_file).extract(vocab_scores)
-            bpe_vocab = {word: i for i, (word, score) in enumerate(vocab_scores)}
-            tokenizer = Tokenizer(
-                BPE(
-                    bpe_vocab,
-                    merges,
-                    unk_token=proto.trainer_spec.unk_piece,
-                    fuse_unk=True,
-                    byte_fallback=self.handle_byte_fallback,
-                    dropout=None,
-                )
-            )
-        else:
-            raise Exception(
-                "You're trying to run a `Unigram` model but you're file was trained with a different algorithm"
-            )
+            return self._unk_id_fn()
+        return 0  # Default unk_id
 
-        # Add special tokens
-        spm_added_tokens = [
-            (id, p.piece, p.type == 3 or p.piece in self.special_tokens)
-            for id, p in enumerate(proto.pieces)
-            if p.type in [3, 4]
-        ]
-        tokenizer.add_tokens(
-            [
-                AddedToken(token, normalized=False, special=special)
-                for id, token, special in sorted(spm_added_tokens, key=lambda x: x[0])
-            ]
-        )
-
-        return tokenizer
+    def tokenizer(self):
+        # Always create empty trainable tokenizer
+        minimal_vocab = [("<unk>", 0.0)]
+        return Tokenizer(Unigram(minimal_vocab, unk_id=self.unk_id(), byte_fallback=self.handle_byte_fallback))
 
-    def normalizer(self, proto):
+    def normalizer(self):
         if self._normalizer_fn is not None:
-            return self._normalizer_fn(proto)
-        
-        precompiled_charsmap = proto.normalizer_spec.precompiled_charsmap
+            return self._normalizer_fn()
         _normalizers = [
             normalizers.Strip(left=False, right=True),
             normalizers.Replace(Regex(" {2,}"), "▁"),
         ]
-        if not precompiled_charsmap:
-            return normalizers.Sequence(_normalizers)
-        else:
-            return normalizers.Sequence([normalizers.Precompiled(precompiled_charsmap)] + _normalizers)
+        return normalizers.Sequence(_normalizers)
 
     def pre_tokenizer(self, replacement, add_prefix_space):
         if self._pre_tokenizer_fn is not None:
@@ -1457,11 +1402,11 @@ def post_processor(self):
         return None
 
     def create_tokenizer(self) -> Tokenizer:
-        """Create and return the configured tokenizer."""
-        tokenizer = self.tokenizer(self.proto)
+        """Create and return the configured empty trainable tokenizer."""
+        tokenizer = self.tokenizer()
 
         # Tokenizer assemble
-        normalizer = self.normalizer(self.proto)
+        normalizer = self.normalizer()
         if normalizer is not None:
             tokenizer.normalizer = normalizer
 
@@ -1483,6 +1428,50 @@ def create_tokenizer(self) -> Tokenizer:
 ## NOTE: LLaMA-specific converter moved to `models/llama/tokenization_llama_fast.py`.
 ## The slow->fast conversion for LLaMA is now handled directly in the fast file.
 
+class LlamaConverter(SpmConverter):
+    handle_byte_fallback = True
+
+    def vocab(self, proto):
+        vocab = [
+            (self.original_tokenizer.convert_ids_to_tokens(0), 0.0),
+            (self.original_tokenizer.convert_ids_to_tokens(1), 0.0),
+            (self.original_tokenizer.convert_ids_to_tokens(2), 0.0),
+        ]
+        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]
+        return vocab
+
+    def unk_id(self, proto):
+        unk_id = 0
+        return unk_id
+
+    def decoder(self, replacement, add_prefix_space):
+        sequence = [
+            decoders.Replace("▁", " "),
+            decoders.ByteFallback(),
+            decoders.Fuse(),
+        ]
+        if add_prefix_space:
+            sequence += [decoders.Strip(content=" ", left=1)]
+        return decoders.Sequence(sequence)
+
+    def normalizer(self, proto):
+        if getattr(self.original_tokenizer, "legacy", True):
+            sequence = []
+            if getattr(self.original_tokenizer, "add_prefix_space", True):
+                sequence += [normalizers.Prepend(prepend="▁")]
+            sequence += [normalizers.Replace(pattern=" ", content="▁")]
+            return normalizers.Sequence(sequence)
+        return None  # non-legacy, no normalizer
+
+    def pre_tokenizer(self, replacement, add_prefix_space):
+        if not getattr(self.original_tokenizer, "legacy", True):  # non-legacy, we need a replace
+            prepend_scheme = _get_prepend_scheme(add_prefix_space, self.original_tokenizer)
+            return pre_tokenizers.Metaspace(replacement=replacement, prepend_scheme=prepend_scheme, split=False)
+        return None
+
+    def post_processor(self):
+        # the processor is defined in the LlamaTokenizerFast class.
+        return None
 
 class MarkupLMConverter(Converter):
     def converted(self) -> Tokenizer:

src/transformers/models/llama/tokenization_llama_fast.py

@@ -97,21 +97,24 @@ class LlamaTokenizerFast(PreTrainedTokenizerFast):
             ```python
             >>> from transformers import LlamaTokenizerFast
 
-            >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=True, from_slow=True)
+            >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=True, from_scratch=True)
             >>> tokenizer.encode("Hello <s>.") # 869 is '▁.'
             [1, 15043, 29871, 1, 869]
             ```
             - `legacy=False`:
             ```python
             >>> from transformers import LlamaTokenizerFast
 
-            >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=False, from_slow=True)
+            >>> tokenizer = LlamaTokenizerFast.from_pretrained("huggyllama/llama-7b", legacy=False, from_scratch=True)
             >>> tokenizer.encode("Hello <s>.")  # 29889 is '.'
             [1, 15043, 29871, 1, 29889]
             ```
             Checkout the [pull request](https://github.com/huggingface/transformers/pull/24565) for more details.
         add_prefix_space (`bool`, *optional*):
             Whether or not the tokenizer should automatically add a prefix space
+        from_scratch (`bool`, *optional*, defaults to `False`):
+            Whether to create an empty trainable tokenizer from scratch. When `True`, creates a minimal tokenizer
+            with only basic special tokens that can be trained on new data.
     """
 
     vocab_files_names = VOCAB_FILES_NAMES
@@ -130,53 +133,32 @@ def __init__(
         add_bos_token=True,
         add_eos_token=False,
         use_default_system_prompt=False,
-        legacy=None,
+        legacy=False,
         add_prefix_space=None,
         **kwargs,
     ):
-        if legacy is None:
-            logger.warning_once(
-                f"You are using the default legacy behaviour of the {self.__class__}. This is"
-                " expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you."
-                " If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it"
-                " means, and thoroughly read the reason why this was added as explained in"
-                " https://github.com/huggingface/transformers/pull/24565 - if you loaded a llama tokenizer from a GGUF file"
-                " you can ignore this message."
-            )
-            legacy = True
-        self.legacy = False
-        legacy = False
+        self.legacy = legacy
 
         # Set add_prefix_space attribute for use in override methods
         self.add_prefix_space = add_prefix_space if add_prefix_space is not None else True
 
-        # Handle from_slow parameter - when True, force SpmTokenizer path even if tokenizer.json exists
-        from_slow = kwargs.pop("from_slow", False)
+        # Handle from_scratch parameter - when True, create empty trainable tokenizer
+        from_scratch = kwargs.pop("from_scratch", False)
 
-        # Handle tokenizer creation
-        if tokenizer_file is not None and not from_slow:
-            # Load from existing tokenizer.json file (unless from_slow=True)
+        if tokenizer_file is not None and not from_scratch:
             from tokenizers import Tokenizer as TokenizerFast
             fast_tokenizer = TokenizerFast.from_file(tokenizer_file)
-        elif vocab_file is not None:
-            # Create LLaMA-specific tokenizer using SpmTokenizer
-            # This path is used when:
-            # 1. vocab_file is provided and no tokenizer_file
-            # 2. from_slow=True (forces SpmTokenizer path even if tokenizer.json exists)
+        else:
             spm_tokenizer = SpmTokenizer(
-                vocab_file=vocab_file,
                 handle_byte_fallback=True,
                 legacy=legacy,
                 add_prefix_space=add_prefix_space if add_prefix_space is not None else True,
                 vocab=self._vocab,
-                #unk_id=self._unk_id,
                 normalizer=self._normalizer,
                 pre_tokenizer=self._pre_tokenizer,
                 decoder=self._decoder,
             )
             fast_tokenizer = spm_tokenizer.create_tokenizer()
-        else:
-            raise ValueError("Either tokenizer_file or vocab_file must be provided")
 
         # Initialize the base class with the fast tokenizer
         super().__init__(
@@ -198,15 +180,14 @@ def __init__(
         self.use_default_system_prompt = use_default_system_prompt
         self.vocab_file = vocab_file
 
-    def _vocab(self, proto):
+    def _vocab(self):
         """Vocabulary handling for this tokenizer."""
         # First 3 special pieces are fixed for LLaMA
         vocab = [
             ("<unk>", 0.0),
             ("<s>", 0.0),
             ("</s>", 0.0),
         ]
-        vocab += [(piece.piece, piece.score) for piece in proto.pieces[3:]]
         return vocab
 
     def _decoder(self, replacement, add_prefix_space):