Skip to content

Commit cd69385

Browse files
Yikunzhangxinyuehfad
Yikun
and
zhangxinyuehfad
authored
Add models test and add serval new models yaml (#3394)
### What this PR does / why we need it? This PR added Add accuracy CI for servals new models - `ascend test / accuracy` is for PR triggered check popluar models accuracy - `ascedn test / models` is for accuracy report, full models test, nightly model test - Add Qwen2-Audio-7B-Instruct, Qwen2-VL-7B-Instruct, Qwen3-8B, Qwen3-VL-30B-A3B-Instruct ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? CI passed Closes: #2330 Closes: #3362 - vLLM version: v0.11.0rc3 - vLLM main: https://github.com/vllm-project/vllm/commit/v0.11.0 --------- Signed-off-by: hfadzxy <[email protected]> Signed-off-by: Yikun Jiang <[email protected]> Co-authored-by: hfadzxy <[email protected]>
1 parent d05d29f commit cd69385

File tree

9 files changed

+434
-285
lines changed

9 files changed

+434
-285
lines changed

‎.github/workflows/_accuracy_test.yaml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
name: 'accuracy test'
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
vllm:
7+
required: true
8+
type: string
9+
vllm-ascend:
10+
required: false
11+
type: string
12+
default: main
13+
runner:
14+
required: true
15+
type: string
16+
image:
17+
required: true
18+
type: string
19+
model_name:
20+
required: true
21+
type: string
22+
upload:
23+
required: false
24+
type: boolean
25+
default: false
26+
27+
jobs:
28+
accuracy_tests:
29+
30+
runs-on: ${{ inputs.runner }}
31+
name: ${{ inputs.model_name }} accuracy
32+
container:
33+
image: swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/cann:8.2.rc1-910b-ubuntu22.04-py3.11
34+
env:
35+
VLLM_USE_MODELSCOPE: True
36+
# 1. If version specified (work_dispatch), do specified branch accuracy test
37+
# 2. If no version (labeled PR), do accuracy test by default ref:
38+
# The branch, tag or SHA to checkout. When checking out the repository that
39+
# triggered a workflow, this defaults to the reference or SHA for that event.
40+
# Otherwise, uses the default branch.
41+
GHA_VLLM_ASCEND_VERSION: ${{ inputs.vllm-ascend }}
42+
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v4
46+
47+
- name: Set model name as output
48+
id: set_output
49+
run: |
50+
echo "model_name=${{ inputs.model_name }}" >> $GITHUB_OUTPUT
51+
52+
- name: Config mirrors
53+
run: |
54+
sed -Ei 's@(ports|archive)[email protected]:8081@g' /etc/apt/sources.list
55+
pip config set global.index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple
56+
pip config set global.trusted-host cache-service.nginx-pypi-cache.svc.cluster.local
57+
apt-get update -y
58+
apt install git -y
59+
60+
- name: Install system dependencies
61+
run: |
62+
apt-get -y install `cat packages.txt`
63+
apt-get -y install gcc g++ cmake libnuma-dev
64+
65+
- name: Checkout vllm-project/vllm repo
66+
uses: actions/checkout@v4
67+
with:
68+
repository: vllm-project/vllm
69+
ref: ${{ inputs.vllm }}
70+
path: ./vllm-empty
71+
72+
- name: Install vllm-project/vllm from source
73+
working-directory: ./vllm-empty
74+
run: |
75+
VLLM_TARGET_DEVICE=empty pip install -e .
76+
77+
- name: Resolve vllm-ascend version
78+
run: |
79+
VERSION_INPUT="${{ inputs.vllm-ascend }}"
80+
81+
if [[ "$VERSION_INPUT" == "latest" ]]; then
82+
TAGS=$(git ls-remote --tags --sort=-v:refname https://github.com/vllm-project/vllm-ascend "v*" | cut -f2 | sed 's|refs/tags/||')
83+
LATEST_TAG=$(echo "$TAGS" | head -n1)
84+
if [[ -z "$LATEST_TAG" ]]; then
85+
RESOLVED_VERSION="main"
86+
else
87+
RESOLVED_VERSION="$LATEST_TAG"
88+
fi
89+
else
90+
RESOLVED_VERSION="$VERSION_INPUT"
91+
fi
92+
echo "GHA_VLLM_ASCEND_VERSION=$RESOLVED_VERSION" >> $GITHUB_ENV
93+
94+
- name: Checkout vllm-project/vllm-ascend repo
95+
uses: actions/checkout@v4
96+
with:
97+
repository: vllm-project/vllm-ascend
98+
path: ./vllm-ascend
99+
ref: ${{ env.GHA_VLLM_ASCEND_VERSION }}
100+
101+
- name: Install vllm-project/vllm-ascend
102+
working-directory: ./vllm-ascend
103+
env:
104+
PIP_EXTRA_INDEX_URL: https://mirrors.huaweicloud.com/ascend/repos/pypi
105+
run: |
106+
pip install -r requirements-dev.txt
107+
pip install -v -e .
108+
109+
- name: Get vLLM commit hash and URL
110+
working-directory: ./vllm-empty
111+
run: |
112+
VLLM_COMMIT=$(git rev-parse --short=7 HEAD)
113+
echo "VLLM_COMMIT=$VLLM_COMMIT" >> $GITHUB_ENV
114+
115+
- name: Get vLLM-Ascend commit hash and URL
116+
working-directory: ./vllm-ascend
117+
run: |
118+
VLLM_ASCEND_COMMIT=$(git rev-parse --short=7 HEAD)
119+
echo "VLLM_ASCEND_COMMIT=$VLLM_ASCEND_COMMIT" >> $GITHUB_ENV
120+
121+
- name: Collect version info
122+
run: |
123+
for dir in /usr/local/Ascend/ascend-toolkit/*; do
124+
dname=$(basename "$dir")
125+
if [ "$dname" != "latest" ]; then
126+
TOOLKIT_DIR="$dname"
127+
break
128+
fi
129+
done
130+
INFO_FILE="/usr/local/Ascend/ascend-toolkit/${TOOLKIT_DIR}/$(uname -i)-linux/ascend_toolkit_install.info"
131+
GHA_CANN_VERSION=$(grep "version=" "$INFO_FILE" \
132+
| head -n1 \
133+
| cut -d'=' -f2 \
134+
| tr -d '"')
135+
{
136+
echo "GHA_CANN_VERSION=$GHA_CANN_VERSION"
137+
pip show torch | grep "Version:" | awk '{print "GHA_TORCH_VERSION="$2}'
138+
pip show torch_npu | grep "Version:" | awk '{print "GHA_TORCH_NPU_VERSION="$2}'
139+
pip show vllm | grep "Version:" | awk '{print "GHA_VLLM_VERSION="$2}' | sed 's/+.*//'
140+
} >> "$GITHUB_ENV"
141+
142+
- name: Run accuracy test
143+
id: report
144+
env:
145+
VLLM_WORKER_MULTIPROC_METHOD: spawn
146+
VLLM_USE_MODELSCOPE: True
147+
VLLM_VERSION: ${{ env.GHA_VLLM_VERSION }}
148+
VLLM_COMMIT: ${{ env.VLLM_COMMIT }}
149+
VLLM_ASCEND_VERSION: ${{ env.GHA_VLLM_ASCEND_VERSION || github.ref }}
150+
VLLM_ASCEND_COMMIT: ${{ env.VLLM_ASCEND_COMMIT }}
151+
CANN_VERSION: ${{ env.GHA_CANN_VERSION }}
152+
TORCH_VERSION: ${{ env.GHA_TORCH_VERSION }}
153+
TORCH_NPU_VERSION: ${{ env.GHA_TORCH_NPU_VERSION }}
154+
run: |
155+
model_base_name=$(basename ${{ inputs.model_name }})
156+
markdown_name="${model_base_name}"
157+
echo "markdown_name=$markdown_name" >> $GITHUB_OUTPUT
158+
mkdir -p ./benchmarks/accuracy
159+
pytest -sv ./tests/e2e/models/test_lm_eval_correctness.py \
160+
--config ./tests/e2e/models/configs/${{ inputs.model_name }}.yaml
161+
162+
- name: Generate step summary
163+
if: ${{ always() }}
164+
run: |
165+
cat ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md >> $GITHUB_STEP_SUMMARY
166+
167+
- name: Upload Report
168+
if: ${{ inputs.upload == true }}
169+
uses: actions/upload-artifact@v4
170+
with:
171+
name: "report-${{ env.GHA_VLLM_ASCEND_VERSION }}-${{ steps.report.outputs.markdown_name }}"
172+
path: ./benchmarks/accuracy/${{ steps.report.outputs.markdown_name }}.md
173+
if-no-files-found: warn
174+
retention-days: 90
175+
overwrite: true

0 commit comments

Comments
 (0)