add eval config for Qwen3-235B-A22B-Thinking-2507-FP8

Signed-off-by: Huamin Li <[email protected]>

Author: hl475

.buildkite/lm-eval-harness/configs/Qwen3-235B-A22B-Thinking-2507-FP8.yaml

@@ -0,0 +1,12 @@
+model_name: "Qwen/Qwen3-235B-A22B-Thinking-2507-FP8"
+backend: "vllm"
+tasks:
+  - name: "gsm8k"
+    metrics:
+      - name: "exact_match,strict-match"
+        value: 0.8
+num_fewshot: 5
+limit: 1000
+max_model_len: 8096
+gen_kwargs: "top_p=1,top_k=0,max_gen_toks=1536"
+apply_chat_template: true

.buildkite/lm-eval-harness/configs/models-medium-h100.txt

@@ -0,0 +1 @@
+Qwen3-235B-A22B-Thinking-2507-FP8.yaml

.buildkite/lm-eval-harness/test_lm_eval_correctness.py

@@ -37,10 +37,14 @@ def launch_lm_eval(eval_config, tp_size):
         limit=eval_config["limit"],
         # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
         # text models. however, this is regressing measured strict-match for
-        # existing text models in CI, so only apply it for mm.
-        apply_chat_template=backend == "vllm-vlm",
+        # existing text models in CI, so only apply it for mm or specified in config.
+        apply_chat_template=eval_config.get(
+            "apply_chat_template", backend == "vllm-vlm"
+        ),
         batch_size=batch_size,
+        gen_kwargs=eval_config.get("gen_kwargs", None),
     )
+
     return results
 
 

.buildkite/test-pipeline.yaml

@@ -1084,7 +1084,7 @@ steps:
   - tests/weight_loading
   commands:
     - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large.txt
-  
+
 - label: NixlConnector PD accuracy tests (Distributed) # 30min
   timeout_in_minutes: 30
   working_dir: "/vllm-workspace/tests"
@@ -1126,6 +1126,18 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
 
+##### H100 test #####
+- label: LM Eval Medium Models (H100) # optional
+  gpu: h100
+  optional: true
+  num_gpus: 4
+  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+  source_file_dependencies:
+  - csrc/
+  - vllm/model_executor/layers/quantization
+  commands:
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-medium-h100.txt --tp-size=4
+
 ##### H200 test #####
 - label: Distributed Tests (H200) # optional
   gpu: h200