Skip to content

Commit 5fd20dd

Browse files
zhangxinyuehfadzhangxinyuehfad
committed
[Test] Add accuracy test report
Signed-off-by: hfadzxy <[email protected]>
1 parent fe13cd9 commit 5fd20dd

File tree

6 files changed

+814
-2
lines changed

6 files changed

+814
-2
lines changed

‎.github/workflows/accuracy_report.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
#
2+
# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
# This file is a part of the vllm-ascend project.
16+
#
17+
18+
name: accuarcy report
19+
on:
20+
workflow_run:
21+
workflows: ["accuarcy test"]
22+
types: [completed]
23+
workflow_dispatch:
24+
inputs:
25+
branch:
26+
description: 'choose a dev branch to pr'
27+
required: true
28+
jobs:
29+
download:
30+
runs-on: ubuntu-latest
31+
steps:
32+
33+
- name: Validate branch name
34+
id: validate_branch
35+
run: |
36+
if [[ "${{ github.event.inputs.branch }}" =~ -dev$ ]]; then
37+
echo "Branch name is valid."
38+
else
39+
echo "Error: The selected branch does not end with '-dev'."
40+
exit 1
41+
fi
42+
43+
- name: Checkout repository
44+
uses: actions/checkout@v4
45+
with:
46+
ref: ${{ github.event.inputs.branch }}
47+
48+
- name: Checkout vllm-project/vllm-ascend repo
49+
uses: actions/checkout@v4
50+
with:
51+
repository: vllm-project/vllm-ascend
52+
path: ./vllm-ascend
53+
fetch-depth: 0
54+
55+
- name: Debug List Artifacts
56+
run: gh api /repos/${{ github.repository }}/actions/artifacts
57+
env:
58+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
59+
60+
- name: Query artifact run id for Qwen2.5-VL-7B-Instruct latest artifact
61+
id: get_Qwen2_5_VL_7B_Instruct_latest_run_id
62+
run: |
63+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
64+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
65+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.branch }}-Qwen2.5-VL-7B-Instruct-accuracy-report")] | sort_by(.created_at) | last | .workflow_run.id')
66+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
67+
env:
68+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
69+
70+
- name: Query artifact run id for latest artifact
71+
id: get_Qwen2_5_7B_Instruct_latest_run_id
72+
run: |
73+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
74+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
75+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.branch }}-Qwen2.5-7B-Instruct-accuracy-report")] | sort_by(.created_at) | last | .workflow_run.id')
76+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
77+
env:
78+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
79+
80+
- name: Query artifact run id for Llama-3.1-8B-Instruct latest artifact
81+
id: get_Llama_3_1_8B_Instruct_latest_run_id
82+
run: |
83+
ARTIFACT_JSON=$(gh api "repos/${{ github.repository }}/actions/artifacts")
84+
RUN_ID=$(echo "$ARTIFACT_JSON" | \
85+
jq -r '[.artifacts[] | select(.name=="${{ github.event.inputs.branch }}-Llama-3.1-8B-Instruct-accuracy-report")] | sort_by(.created_at) | last | .workflow_run.id')
86+
echo "runid=$RUN_ID" >> "$GITHUB_OUTPUT"
87+
env:
88+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
89+
90+
- name: Download Qwen/Qwen2.5-VL-7B-Instruct Artifact
91+
uses: actions/download-artifact@v4
92+
with:
93+
name: ${{ github.event.inputs.branch }}-Qwen2.5-VL-7B-Instruct-accuracy-report
94+
path: ./docs/source/developer_guide/evaluation/accuracy_report
95+
github-token: ${{ secrets.GITHUB_TOKEN }}
96+
repository: vllm-project/vllm-ascend
97+
run-id: ${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id.outputs.runid }}
98+
99+
- name: Download Qwen/Qwen2.5-7B-Instruct Artifact
100+
uses: actions/download-artifact@v4
101+
with:
102+
name: ${{ github.event.inputs.branch }}-Qwen2.5-7B-Instruct-accuracy-report
103+
path: ./docs/source/developer_guide/evaluation/accuracy_report
104+
github-token: ${{ secrets.GITHUB_TOKEN }}
105+
repository: vllm-project/vllm-ascend
106+
run-id: ${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id.outputs.runid }}
107+
108+
- name: Download meta-llama/Llama-3.1-8B-Instruct Artifact
109+
uses: actions/download-artifact@v4
110+
with:
111+
name: ${{ github.event.inputs.branch }}-Llama-3.1-8B-Instruct-accuracy-report
112+
path: ./docs/source/developer_guide/evaluation/accuracy_report
113+
github-token: ${{ secrets.GITHUB_TOKEN }}
114+
repository: vllm-project/vllm-ascend
115+
run-id: ${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id.outputs.runid }}
116+
117+
- name: Display Files
118+
working-directory: ./docs/source/developer_guide/evaluation/accuracy_report
119+
run: |
120+
cat ./Qwen2.5-VL-7B-Instruct.md
121+
cat ./Llama-3.1-8B-Instruct.md
122+
cat ./Qwen2.5-7B-Instruct.md
123+
124+
- name: Create Pull Request for markdown update
125+
uses: peter-evans/create-pull-request@v7
126+
with:
127+
token: ${{ secrets.PR_TOKEN }}
128+
base: ${{ env.SANITIZED_BRANCH_NAME }}
129+
branch: auto-pr/accuracy-test
130+
commit-message: "Update accuracy report for ${{ github.event.inputs.branch }}"
131+
add-paths: ./docs/source/developer_guide/evaluation/accuracy_report/*.md
132+
title: "[Doc]Update accuracy report for ${{ github.event.inputs.branch }}"
133+
body: |
134+
The accuracy results running on Ascend NPU have changed, I'm updating the report.
135+
Please review the changes.
136+
137+
- [Workflow run][1]
138+
- [Qwen2.5-7B-Instruct accuracy report][2]
139+
- [Llama-3.1-8B-Instruct accuracy report][3]
140+
- [Qwen2.5-VL-7B-Instruct accuracy report][4]
141+
142+
[1]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
143+
[2]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_7B_Instruct_latest_run_id.outputs.runid }}
144+
[3]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Llama_3_1_8B_Instruct_latest_run_id.outputs.runid }}
145+
[4]: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ steps.get_Qwen2_5_VL_7B_Instruct_latest_run_id.outputs.runid }}

0 commit comments

Comments
 (0)