update

Signed-off-by: zhewenli <[email protected]>

Author: zhewenl

.buildkite/lm-eval-harness/configs/DeepSeek-R1.yaml

@@ -1,10 +1,15 @@
 # For vllm script, with -t option (tensor parallel size).
-# bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m deepseek-ai/DeepSeek-R1 -l 250 -t 8 -f 5
+# bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m deepseek-ai/DeepSeek-R1 -b 128 -l 250 -f 5 -t 8
 model_name: "deepseek-ai/DeepSeek-R1"
 tasks:
 - name: "mmlu_pro"
   metrics:
   - name: "exact_match,custom-extract"
-    value: 0.84
-limit: 250 # will run on 250 * 14 subjects = 3500 samples
+    value: 0.80
+limit: 2 # will run on 250 * 14 subjects = 3500 samples
 num_fewshot: 5
+trust_remote_code: True
+rtol: 0.05
+vllm_args:
+  enable_expert_parallel: true
+  gpu_memory_utilization: 0.95

.buildkite/lm-eval-harness/configs/models-large-h100.txt

@@ -1 +1 @@
-Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
+DeepSeek-R1.yaml

.buildkite/lm-eval-harness/test_lm_eval_correctness.py

@@ -13,22 +13,32 @@
 import numpy as np
 import yaml
 
-RTOL = 0.08
+DEFAULT_RTOL = 0.08
 
 
 def launch_lm_eval(eval_config, tp_size):
     trust_remote_code = eval_config.get("trust_remote_code", False)
     max_model_len = eval_config.get("max_model_len", 4096)
     batch_size = eval_config.get("batch_size", "auto")
     backend = eval_config.get("backend", "vllm")
-    model_args = (
-        f"pretrained={eval_config['model_name']},"
-        f"tensor_parallel_size={tp_size},"
-        f"enforce_eager=true,"
-        f"add_bos_token=true,"
-        f"trust_remote_code={trust_remote_code},"
-        f"max_model_len={max_model_len},"
-    )
+
+    model_args_list = [
+        f"pretrained={eval_config['model_name']}",
+        f"tensor_parallel_size={tp_size}",
+        "enforce_eager=true",
+        "add_bos_token=true",
+        f"trust_remote_code={trust_remote_code}",
+        f"max_model_len={max_model_len}",
+    ]
+
+    if "vllm_args" in eval_config:
+        for key, value in eval_config["vllm_args"].items():
+            if isinstance(value, bool):
+                value = str(value).lower()
+            model_args_list.append(f"{key}={value}")
+
+    model_args = ",".join(model_args_list)
+
     results = lm_eval.simple_evaluate(
         model=backend,
         model_args=model_args,
@@ -49,15 +59,18 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
 
     results = launch_lm_eval(eval_config, tp_size)
 
+    rtol = eval_config.get("rtol", DEFAULT_RTOL)
+
     success = True
     for task in eval_config["tasks"]:
         for metric in task["metrics"]:
             ground_truth = metric["value"]
             measured_value = results["results"][task["name"]][metric["name"]]
             print(
                 f"{task['name']} | {metric['name']}: "
-                f"ground_truth={ground_truth} | measured={measured_value}"
+                f"ground_truth={ground_truth:.3f} | "
+                f"measured={measured_value:.3f} | rtol={rtol}"
             )
-            success = success and np.isclose(ground_truth, measured_value, rtol=RTOL)
+            success = success and np.isclose(ground_truth, measured_value, rtol=rtol)
 
     assert success