fix mistake tokenizer

baptiste-aubertin · baptiste-aubertin · commit 1f91d312c047 · 2025-11-24T11:39:24.000+01:00
diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py
@@ -366,6 +366,8 @@
         ("layoutlmv3", ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast" if is_tokenizers_available() else None)),
         ("layoutxlm", ("LayoutXLMTokenizer", "LayoutXLMTokenizerFast" if is_tokenizers_available() else None)),
         ("led", ("LEDTokenizer", "LEDTokenizerFast" if is_tokenizers_available() else None)),
+        ("lfm2", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
+        ("lfm2_vl", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),
         (
             "lightonocr",
             (

Original file line number	Diff line number	Diff line change
`@@ -366,6 +366,8 @@`
`366`	`366`	`("layoutlmv3", ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast" if is_tokenizers_available() else None)),`
`367`	`367`	`("layoutxlm", ("LayoutXLMTokenizer", "LayoutXLMTokenizerFast" if is_tokenizers_available() else None)),`
`368`	`368`	`("led", ("LEDTokenizer", "LEDTokenizerFast" if is_tokenizers_available() else None)),`
	`369`	`+ ("lfm2", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),`
	`370`	`+ ("lfm2_vl", (None, "PreTrainedTokenizerFast" if is_tokenizers_available() else None)),`
`369`	`371`	`(`
`370`	`372`	`"lightonocr",`
`371`	`373`	`(`