wx44wx/ZNJJ-api-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(prompts): refine vehicle count review and clarification replies

Browse Source
This commit is contained in:
Eric Wang
2026-10-28 18:17:07 +08:00
parent 1fc72a704d
commit f1277e8413
4 changed files with 49 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/prompt_probes.py
+15 -6
View File
@@ -98,8 +98,8 @@ def probes():
code='2000', count=1, not_matches='(几辆|多少辆|其他.*车.*吗|这个时间对吗)'),
dict(name='feedback_single_motor_complaint', user=complaint, history=[
{'role':'user','content':'我的小轿车撞到路边围栏，没有人受伤，没有非机动车或行人，我还在现场。'},
*confirmed, count_question], code='2000', count=1,
not_matches='(几辆|多少辆|没.*看到|其他.*车.*吗)'),
*confirmed, count_question], code=['1002', '2000'], count=1,
not_matches='(没.*(看到|收到)|这个时间对吗)'),
dict(name='holdout_single_motor_different_object', user='我的货车倒车碰到停车场立柱，车受损了。',
history=[], code='1002', count=1, contains='受伤'),
dict(name='holdout_only_vehicle_type_not_count', user='我开的是自己的小轿车。',
@@ -111,12 +111,12 @@ def probes():
not_matches='(没.*(看到|收到)|未.*(提供|回答)|具体的数字)'),
dict(name='feedback_known_count_in_history', user=complaint,
history=[{'role': 'user', 'content': '只有我这一辆小轿车撞到围栏。没有人受伤，没有非机动车或行人，我还在现场。'}, *confirmed, count_question],
code='2000', count=1, not_matches='(几辆|多少辆|具体的数字|没.*看到)'),
code=['1002', '2000'], count=1, not_matches='(没.*(看到|收到)|这个时间对吗)'),
dict(name='feedback_reverse_collision_extracts_one', user='倒车时碰到了停车场墙面，车刮坏了。',
history=[], code='1002', count=1, contains='受伤'),
dict(name='feedback_form_two_skips_count', user=complaint,
history=[*confirmed, count_question], form={**known_form, 'vehicle_count': 2},
code='2010', count=2, not_matches='(几辆|多少辆|是否.*两辆|确认.*数量|没.*看到)'),
code=['1002', '2010'], count=2, not_matches='(没.*(看到|收到)|这个时间对吗)'),
dict(name='feedback_form_three_skips_count', user=complaint,
history=[*confirmed, count_question], form={**known_form, 'vehicle_count': 3},
code='0003', count=3, not_matches='(几辆|多少辆|确认.*数量|没.*看到)'),
@@ -127,7 +127,13 @@ def probes():
not_matches='(几辆|多少辆|什么.*车型|是否.*一辆|只有.*一辆.*吗)'),
dict(name='feedback_known_count_in_form', user=complaint,
history=[*confirmed, count_question], form={**known_form, 'vehicle_count': 1},
code='2000', not_matches='(几辆|多少辆|具体的数字|没.*看到)'),
code=['1002', '2000'], not_matches='(没.*(看到|收到)|这个时间对吗)'),
*[dict(name='feedback_count_review_answer_' + str(count), user='对，数量没错。',
history=[*confirmed, {'role': 'assistant', 'content':
f'<state>1002</state>我这边记录到的是{count}辆汽车参与，请问这个数量对吗？'}],
form={**known_form, 'vehicle_count': count}, code=target, count=count,
not_matches='(几辆|多少辆|数量对吗|这个时间对吗)')
for count, target in [(1, '2000'), (2, '2010')]],
dict(name='feedback_form_false_and_unknown_count', user=complaint,
history=[*confirmed, count_question], form=known_form,
code='1002', count_absent=True,
@@ -149,6 +155,7 @@ def probes():
not_matches='(简单.*(描述|说一下).*经过|能听到|听得.*清)'),
dict(name='feedback_no_evidence_of_previous_answer', user=complaint,
history=[count_question], code='1002', count_absent=True,
matches='^这项再帮我核对一下：这次是您这一辆汽车，还是还有其他汽车参与？$',
not_matches='(已记录.*一辆|已确认.*一辆|没.*看到|具体的数字)'),
dict(name='partial_symptom_negation', user='没有流血。', history=injury_q,
code='1002', injury=None, contains='受伤'),
@@ -211,7 +218,9 @@ async def main(args):
raw = await llm.complete(render_prompt(COLLECTION, state, now=NOW), user_message, history=case['history'])
code, body = parse_state_reply(raw)
evidence['reply'] = raw
checks.append(dict(text='expected state '+case['code'], passed=code==case['code'], evidence=raw))
expected_codes = case['code'] if isinstance(case['code'], list) else [case['code']]
checks.append(dict(text='expected state ' + ' or '.join(expected_codes),
passed=code in expected_codes, evidence=raw))
if 'contains' in case:
checks.append(dict(text='asks/mentions '+case['contains'], passed=case['contains'] in body, evidence=body))
for key, negate in [('matches', False), ('not_matches', True)]: