84 lines
4.3 KiB
Python
84 lines
4.3 KiB
Python
|
"""Live end-to-end time provenance probes, separate from unchanged cases."""
|
||
|
|
import argparse
|
||
|
|
import asyncio
|
||
|
|
from datetime import datetime, timedelta
|
||
|
|
import json
|
||
|
|
from pathlib import Path
|
||
|
|
import re
|
||
|
|
from zoneinfo import ZoneInfo
|
||
|
|
|
||
|
|
from src.agent.collect import run_collect
|
||
|
|
from src.agent.llm import build_llm_service
|
||
|
|
|
||
|
|
TZ = ZoneInfo('Asia/Shanghai')
|
||
|
|
ORIGIN = datetime(2026,10,22,0,5,tzinfo=TZ)
|
||
|
|
HISTORY = [
|
||
|
|
{'role':'user','content':'十分钟前两辆汽车追尾，没有非机动车或行人，我还在现场。'},
|
||
|
|
{'role':'assistant','content':'<state>1002</state>请问有没有人员受伤？'},
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
def parsed_time(value):
|
||
|
|
if not isinstance(value,str):
|
||
|
|
return None
|
||
|
|
match = re.fullmatch(r'(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2})点(\d{1,2})分',value)
|
||
|
|
return tuple(map(int,match.groups())) if match else None
|
||
|
|
|
||
|
|
|
||
|
|
async def main(output):
|
||
|
|
llm = build_llm_service('openai')
|
||
|
|
cases = []
|
||
|
|
for delay in (75,135):
|
||
|
|
cases.append(dict(name=f'anchored_cross_midnight_delay_{delay}',now=ORIGIN+timedelta(minutes=delay),
|
||
|
|
input='没有人受伤。',clocks={'0':ORIGIN.isoformat()},expected=(2026,10,21,23,55)))
|
||
|
|
for repeat in range(5):
|
||
|
|
cases.append(dict(name=f'legacy_missing_reference_{repeat}',now=ORIGIN+timedelta(hours=2),
|
||
|
|
input='没有人受伤。',clocks={},expected=None))
|
||
|
|
cases.append(dict(name='explicit_absolute_correction',now=ORIGIN+timedelta(hours=2),
|
||
|
|
input='没有人受伤，事故时间更正为2026年10月21日23点40分。',
|
||
|
|
clocks={'0':ORIGIN.isoformat()},expected=(2026,10,21,23,40)))
|
||
|
|
semaphore=asyncio.Semaphore(4)
|
||
|
|
async def run(case):
|
||
|
|
calls=[]
|
||
|
|
class Recorder:
|
||
|
|
async def complete(self,system_prompt,user_message,*,history=None):
|
||
|
|
raw=await llm.complete(system_prompt,user_message,history=history)
|
||
|
|
calls.append({'kind':'extraction' if system_prompt.startswith('【字段提取】') else 'dialogue',
|
||
|
|
'raw':raw})
|
||
|
|
return raw
|
||
|
|
async with semaphore:
|
||
|
|
checks=[]
|
||
|
|
try:
|
||
|
|
result=await run_collect({'stage_code':'1002','dialogue':HISTORY,
|
||
|
|
'dialogue_timestamps':case['clocks'],'input_text':case['input']},
|
||
|
|
Recorder(),now=case['now'])
|
||
|
|
reply=result['response_text']
|
||
|
|
fields=result['accident_fields']
|
||
|
|
checks.append(dict(text='continues collection',passed=result['stage_code']=='1002'))
|
||
|
|
expected=case['expected']
|
||
|
|
if expected is None:
|
||
|
|
checks.append(dict(text='no unanchored time in form',passed='accident_time' not in fields))
|
||
|
|
checks.append(dict(text='asks for time without inventing it',passed=not re.search(r'\d{4}年.*点.*分',reply)
|
||
|
|
and bool(re.search('什么时候|几点|事故.*时间',reply))))
|
||
|
|
else:
|
||
|
|
checks.append(dict(text='form uses original clock or explicit correction',
|
||
|
|
passed=parsed_time(fields.get('accident_time'))==expected))
|
||
|
|
value=re.search(r'\d{4}年\d{1,2}月\d{1,2}日\d{1,2}点\d{1,2}分',reply)
|
||
|
|
checks.append(dict(text='dialogue confirms same supported value',
|
||
|
|
passed=bool(value) and parsed_time(value.group())==expected))
|
||
|
|
return dict(name=case['name'],passed=all(c['passed'] for c in checks),checks=checks,
|
||
|
|
reply=reply,fields=fields,calls=calls)
|
||
|
|
except Exception as exc:
|
||
|
|
return dict(name=case['name'],passed=False,error=type(exc).__name__+': '+str(exc),calls=calls)
|
||
|
|
results=await asyncio.gather(*(run(c) for c in cases))
|
||
|
|
report={'passed':sum(r['passed'] for r in results),'total':len(results),'results':results}
|
||
|
|
output.parent.mkdir(parents=True,exist_ok=True)
|
||
|
|
output.write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
|
||
|
|
print(json.dumps({k:v for k,v in report.items() if k!='results'}))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__=='__main__':
|
||
|
|
parser=argparse.ArgumentParser(__doc__)
|
||
|
|
parser.add_argument('--output',type=Path,required=True)
|
||
|
|
asyncio.run(main(parser.parse_args().output))
|