Transformers serve fix (#42570)

* Fix: lacking EOS token + failing AutoProcessor

* Tests

* Tests

Author: LysandreJik

src/transformers/cli/serve.py

@@ -36,7 +36,7 @@
 from tqdm import tqdm
 
 import transformers
-from transformers import BitsAndBytesConfig, GenerationConfig
+from transformers import AutoTokenizer, BitsAndBytesConfig, GenerationConfig, PreTrainedTokenizerBase
 from transformers.utils.import_utils import (
     is_fastapi_available,
     is_librosa_available,
@@ -823,9 +823,9 @@ def continuous_batching_chat_completion(self, req: dict, request_id: str) -> Str
             self.running_continuous_batching_manager.start()
 
         # TODO (Joao, Lysandre): this should also work with tool support
-        inputs = processor.apply_chat_template(req["messages"], return_tensors="pt", add_generation_prompt=True).to(
-            model.device
-        )["input_ids"][0]
+        inputs = processor.apply_chat_template(
+            req["messages"], return_tensors="pt", add_generation_prompt=True, return_dict=True
+        ).to(model.device)["input_ids"][0]
 
         def stream_chat_completion(request_id, decode_stream):
             from ..generation.continuous_batching import RequestStatus
@@ -841,8 +841,13 @@ def stream_chat_completion(request_id, decode_stream):
 
                     if result.status == RequestStatus.FINISHED:
                         generated_all_tokens = n_tokens_generated >= generation_config.max_new_tokens
-                        final_token_is_eos = result == tokenizer.eos_token
-                        reason = "length" if (generated_all_tokens and not final_token_is_eos) else "stop"
+
+                        # If the tokenizer has an eos_token, we can have a more robust check.
+                        if hasattr(tokenizer, "eos_token"):
+                            final_token_is_eos = result == tokenizer.eos_token
+                            generated_all_tokens = generated_all_tokens and not final_token_is_eos
+
+                        reason = "length" if generated_all_tokens else "stop"
 
                         yield self.build_chat_completion_chunk(
                             request_id,
@@ -921,7 +926,11 @@ def cancellation_wrapper_buffer(_request_id):
             return JSONResponse(json_chunk, media_type="application/json")
 
     @staticmethod
-    def get_model_modality(model: "PreTrainedModel") -> Modality:
+    def get_model_modality(model: "PreTrainedModel", processor=None) -> Modality:
+        if processor is not None:
+            if isinstance(processor, PreTrainedTokenizerBase):
+                return Modality.LLM
+
         from transformers.models.auto.modeling_auto import (
             MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
             MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING_NAMES,
@@ -1011,7 +1020,7 @@ def generate_chat_completion(self, req: dict) -> StreamingResponse | JSONRespons
         self.last_model = model_id_and_revision
         model, processor = self.load_model_and_processor(model_id_and_revision)
 
-        modality = self.get_model_modality(model)
+        modality = self.get_model_modality(model, processor=processor)
         processor_inputs = self.get_processor_inputs_from_inbound_messages(messages, modality)
 
         # ====== TOOL PREPROCESSING LOGIC ======
@@ -1184,8 +1193,14 @@ def generate_with_cache(**kwargs):
                         )
 
                 generated_all_tokens = n_tokens_generated >= generation_config.max_new_tokens
-                final_token_is_eos = result == streamer.tokenizer.eos_token
-                reason = "length" if (generated_all_tokens and not final_token_is_eos) else "stop"
+
+                # If the tokenizer has an eos_token, we can have a more robust check.
+                if hasattr(streamer.tokenizer, "eos_token"):
+                    final_token_is_eos = result == streamer.tokenizer.eos_token
+                    generated_all_tokens = generated_all_tokens and not final_token_is_eos
+
+                reason = "length" if generated_all_tokens else "stop"
+
                 yield self.build_chat_completion_chunk(_request_id, finish_reason=reason, model=model_id_and_revision)
 
                 thread.join()
@@ -1272,7 +1287,9 @@ def generate_response(self, req: dict) -> Generator[str, None, None]:
         else:
             raise TypeError("inputs should be a list, dict, or str")
 
-        inputs = processor.apply_chat_template(inputs, add_generation_prompt=True, return_tensors="pt")["input_ids"]
+        inputs = processor.apply_chat_template(
+            inputs, add_generation_prompt=True, return_tensors="pt", return_dict=True
+        )["input_ids"]
         inputs = inputs.to(model.device)
         request_id = req.get("previous_response_id", "req_0")
 
@@ -1576,7 +1593,9 @@ def generate_response_non_streaming(self, req: dict) -> dict:
         else:
             raise ValueError("inputs should be a list, dict, or str")
 
-        inputs = processor.apply_chat_template(inputs, add_generation_prompt=True, return_tensors="pt")["input_ids"]
+        inputs = processor.apply_chat_template(
+            inputs, add_generation_prompt=True, return_tensors="pt", return_dict=True
+        )["input_ids"]
         inputs = inputs.to(model.device)
         request_id = req.get("previous_response_id", "req_0")
 
@@ -1775,11 +1794,22 @@ def _load_model_and_data_processor(self, model_id_and_revision: str):
         else:
             model_id, revision = model_id_and_revision, "main"
 
-        data_processor = AutoProcessor.from_pretrained(
-            model_id,
-            revision=revision,
-            trust_remote_code=self.trust_remote_code,
-        )
+        try:
+            data_processor = AutoProcessor.from_pretrained(
+                model_id,
+                revision=revision,
+                trust_remote_code=self.trust_remote_code,
+            )
+        except OSError:
+            try:
+                data_processor = AutoTokenizer.from_pretrained(
+                    model_id,
+                    revision=revision,
+                    trust_remote_code=self.trust_remote_code,
+                )
+            except OSError:
+                raise OSError("Failed to load processor with `AutoProcessor` and `AutoTokenizer`.")
+
         dtype = self.dtype if self.dtype in ["auto", None] else getattr(torch, self.dtype)
         quantization_config = self.get_quantization_config()
 

tests/cli/test_serve.py

@@ -329,7 +329,7 @@ def test_generation_config_in_request(self):
 
     def test_early_return_due_to_length(self):
         request = {
-            "model": "Qwen/Qwen3-0.6B",
+            "model": "Qwen/Qwen2.5-0.5B-Instruct",
             "messages": [{"role": "user", "content": "Hello, how are you?"}],
             "stream": True,
             "max_tokens": 3,
@@ -339,8 +339,17 @@ def test_early_return_due_to_length(self):
         last_payload = all_payloads[-1]
         self.assertTrue(last_payload.choices[0]["finish_reason"] == "length")
 
-    # TODO: one test for each request flag, to confirm it is working as expected
-    # TODO: speed-based test to confirm that KV cache is working across requests
+    def test_continues_until_stop(self):
+        request = {
+            "model": "Qwen/Qwen2.5-0.5B-Instruct",
+            "messages": [{"role": "user", "content": 'Please only answer with "Hi."'}],
+            "stream": True,
+            "max_tokens": 30,
+        }
+
+        all_payloads = self.run_server(request)
+        last_payload = all_payloads[-1]
+        self.assertTrue(last_payload.choices[0]["finish_reason"] == "stop")
 
 
 class ServeCompletionsGenerateMockTests(unittest.TestCase):