Skip to content

Commit c1309a3

Browse files
mohi-devhubmohi-devhub
committed
fix: use configurable model and real OmniParser fields
1 parent 4a59df8 commit c1309a3

File tree

2 files changed

+259
-4
lines changed

2 files changed

+259
-4
lines changed

‎app/services/heuristic_engine.py‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ async def _evaluate_with_llm(
234234
try:
235235
# Call LLM
236236
response = await self.llm_client.chat.completions.create(
237-
model="gpt-4o",
237+
model=settings.OPENAI_MODEL,
238238
messages=[
239239
{"role": "system", "content": "You are a UX evaluation expert. Respond only with valid JSON."},
240240
{"role": "user", "content": prompt}
@@ -521,16 +521,22 @@ async def _llm_explain_heuristic(
521521
score: int,
522522
violations: List[HeuristicViolation]
523523
) -> Optional[str]:
524+
"""Generate LLM-based explanation for heuristic evaluation results.
525+
526+
Uses only real OmniParser output fields (type, bbox, interactivity, content).
527+
Gracefully handles missing/optional fields.
528+
"""
524529
if not self.llm_client:
525530
return None
526531

527532
try:
533+
# Serialize elements using only real OmniParser fields
528534
elements_brief = [
529535
{
530536
"type": e.element_type,
531537
"text": e.text,
532-
"interactive": e.interactive,
533-
"attributes": list(e.attributes.keys())
538+
"interactivity": e.interactivity, # Use real field name
539+
"bbox": e.bbox # Include bounding box for context
534540
}
535541
for e in detection_result.elements[:20]
536542
]
@@ -571,5 +577,8 @@ async def _llm_explain_heuristic(
571577
return response.choices[0].message.content.strip() if response.choices else None
572578

573579
except Exception as exc:
574-
self.logger.warning(f"LLM explanation failed for {heuristic_id.value}: {exc}")
580+
self.logger.warning(
581+
f"LLM explanation failed for {heuristic_id.value}: {exc}. "
582+
f"Falling back to empty explanation."
583+
)
575584
return None

‎examples/omniparser_raw_output.json‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
{
2+
"metadata": {
3+
"screenshot_id": "screenshot_001",
4+
"timestamp": "2025-12-09T12:34:56Z",
5+
"device": "desktop",
6+
"resolution": {
7+
"width": 1920,
8+
"height": 1080
9+
},
10+
"omniparser_version": "1.0.0"
11+
},
12+
"elements": [
13+
{
14+
"id": "element_001",
15+
"type": "heading",
16+
"role": "heading",
17+
"text": "Create New Account",
18+
"bbox": [48, 32, 384, 64],
19+
"interactivity": "noninteractive",
20+
"visible": true,
21+
"hierarchy_level": 1,
22+
"attributes": {}
23+
},
24+
{
25+
"id": "element_002",
26+
"type": "text",
27+
"role": "text",
28+
"text": "Enter your email address to get started",
29+
"bbox": [48, 80, 512, 104],
30+
"interactivity": "noninteractive",
31+
"visible": true,
32+
"attributes": {}
33+
},
34+
{
35+
"id": "element_003",
36+
"type": "input",
37+
"role": "textbox",
38+
"text": "[email protected]",
39+
"bbox": [48, 120, 448, 160],
40+
"interactivity": "interactive",
41+
"visible": true,
42+
"input_type": "email",
43+
"placeholder": "Email address",
44+
"required": true,
45+
"attributes": {
46+
"name": "email",
47+
"type": "email"
48+
}
49+
},
50+
{
51+
"id": "element_004",
52+
"type": "input",
53+
"role": "textbox",
54+
"text": "••••••••••",
55+
"bbox": [48, 176, 448, 216],
56+
"interactivity": "interactive",
57+
"visible": true,
58+
"input_type": "password",
59+
"placeholder": "Password",
60+
"required": true,
61+
"attributes": {
62+
"name": "password",
63+
"type": "password"
64+
}
65+
},
66+
{
67+
"id": "element_005",
68+
"type": "label",
69+
"role": "label",
70+
"text": "I agree to the Terms of Service",
71+
"bbox": [48, 232, 336, 256],
72+
"interactivity": "noninteractive",
73+
"visible": true,
74+
"attributes": {}
75+
},
76+
{
77+
"id": "element_006",
78+
"type": "checkbox",
79+
"role": "checkbox",
80+
"text": "",
81+
"bbox": [30, 232, 50, 252],
82+
"interactivity": "interactive",
83+
"visible": true,
84+
"checked": false,
85+
"attributes": {
86+
"name": "terms_agreement"
87+
}
88+
},
89+
{
90+
"id": "element_007",
91+
"type": "button",
92+
"role": "button",
93+
"text": "Create Account",
94+
"bbox": [48, 280, 240, 320],
95+
"interactivity": "interactive",
96+
"visible": true,
97+
"button_type": "submit",
98+
"attributes": {
99+
"type": "submit",
100+
"aria-label": "Create Account Button"
101+
}
102+
},
103+
{
104+
"id": "element_008",
105+
"type": "button",
106+
"role": "button",
107+
"text": "Cancel",
108+
"bbox": [256, 280, 448, 320],
109+
"interactivity": "interactive",
110+
"visible": true,
111+
"button_type": "reset",
112+
"attributes": {
113+
"type": "reset",
114+
"aria-label": "Cancel Button"
115+
}
116+
},
117+
{
118+
"id": "element_009",
119+
"type": "link",
120+
"role": "link",
121+
"text": "Already have an account? Sign in",
122+
"bbox": [48, 352, 336, 376],
123+
"interactivity": "interactive",
124+
"visible": true,
125+
"href": "/login",
126+
"attributes": {
127+
"href": "/login"
128+
}
129+
},
130+
{
131+
"id": "element_010",
132+
"type": "div",
133+
"role": "region",
134+
"text": "Security Tips",
135+
"bbox": [480, 120, 1120, 360],
136+
"interactivity": "noninteractive",
137+
"visible": true,
138+
"children": [
139+
{
140+
"id": "element_010_1",
141+
"type": "heading",
142+
"role": "heading",
143+
"text": "Security Tips",
144+
"bbox": [480, 120, 720, 152],
145+
"interactivity": "noninteractive",
146+
"visible": true,
147+
"hierarchy_level": 2,
148+
"attributes": {}
149+
},
150+
{
151+
"id": "element_010_2",
152+
"type": "text",
153+
"role": "text",
154+
"text": "• Use a strong password with mixed case and numbers",
155+
"bbox": [480, 160, 1080, 184],
156+
"interactivity": "noninteractive",
157+
"visible": true,
158+
"attributes": {}
159+
},
160+
{
161+
"id": "element_010_3",
162+
"type": "text",
163+
"role": "text",
164+
"text": "• Enable two-factor authentication for extra security",
165+
"bbox": [480, 192, 1080, 216],
166+
"interactivity": "noninteractive",
167+
"visible": true,
168+
"attributes": {}
169+
},
170+
{
171+
"id": "element_010_4",
172+
"type": "text",
173+
"role": "text",
174+
"text": "• Never share your password with anyone",
175+
"bbox": [480, 224, 1080, 248],
176+
"interactivity": "noninteractive",
177+
"visible": true,
178+
"attributes": {}
179+
}
180+
],
181+
"attributes": {}
182+
},
183+
{
184+
"id": "element_011",
185+
"type": "icon",
186+
"role": "presentation",
187+
"text": "checkmark",
188+
"bbox": [1152, 160, 1192, 200],
189+
"interactivity": "noninteractive",
190+
"visible": true,
191+
"icon_type": "success",
192+
"attributes": {
193+
"aria-hidden": "true"
194+
}
195+
}
196+
],
197+
"layout_tree": {
198+
"id": "root",
199+
"children": [
200+
{
201+
"id": "element_001",
202+
"bbox": [48, 32, 384, 64]
203+
},
204+
{
205+
"id": "element_002",
206+
"bbox": [48, 80, 512, 104]
207+
},
208+
{
209+
"id": "element_003",
210+
"bbox": [48, 120, 448, 160]
211+
},
212+
{
213+
"id": "element_004",
214+
"bbox": [48, 176, 448, 216]
215+
},
216+
{
217+
"id": "element_005",
218+
"bbox": [48, 232, 336, 256]
219+
},
220+
{
221+
"id": "element_006",
222+
"bbox": [30, 232, 50, 252]
223+
},
224+
{
225+
"id": "element_007",
226+
"bbox": [48, 280, 240, 320]
227+
},
228+
{
229+
"id": "element_008",
230+
"bbox": [256, 280, 448, 320]
231+
},
232+
{
233+
"id": "element_009",
234+
"bbox": [48, 352, 336, 376]
235+
},
236+
{
237+
"id": "element_010",
238+
"bbox": [480, 120, 1120, 360]
239+
},
240+
{
241+
"id": "element_011",
242+
"bbox": [1152, 160, 1192, 200]
243+
}
244+
]
245+
}
246+
}

0 commit comments

Comments
 (0)