Eric Wang
e5d2a78886
- Config.llm_profiles: LLM_PROVIDERS declares names, each group defined by LLM_<NAME>_BASE_URL/_API_KEY/_MODEL (required) and optional _TIMEOUT_SECONDS/ _MAX_RETRIES/_TEMPERATURE/_EXTRA_BODY; JSON-dict form of LLM_PROVIDERS also supported. LLM_PROVIDER selects the active group at runtime; empty/fake = offline scripted model - removed fixed openai/kimi groups (AGENT_LLM_PROVIDER, KIMI_LLM_*) - validate(): incomplete groups and unknown LLM_PROVIDER fail fast with the configured-name list - OpenAILLMService: http client trust_env from LLM_HTTP_TRUST_ENV (default False - local gateways must bypass the Windows system proxy) - build_llm_service(provider): explicit name or LLM_PROVIDER default - .env/.env.example: deepseek + kimi groups side by side, switch by LLM_PROVIDER=kimi (thinking off via EXTRA_BODY reasoning_effort=none) - eval probe scripts updated to the new config interface Verified: 172 unit tests; live calls succeed on both providers (deepseek 28s, kimi 6.6s, both emitting valid <state> prefixes)
84 lines
4.3 KiB
Python
84 lines
4.3 KiB
Python
"""Live end-to-end time provenance probes, separate from unchanged cases."""
|
|
import argparse
|
|
import asyncio
|
|
from datetime import datetime, timedelta
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from src.agent.collect import run_collect
|
|
from src.agent.llm import build_llm_service
|
|
|
|
TZ = ZoneInfo('Asia/Shanghai')
|
|
ORIGIN = datetime(2026,10,22,0,5,tzinfo=TZ)
|
|
HISTORY = [
|
|
{'role':'user','content':'十分钟前两辆汽车追尾，没有非机动车或行人，我还在现场。'},
|
|
{'role':'assistant','content':'<state>1002</state>请问有没有人员受伤？'},
|
|
]
|
|
|
|
|
|
def parsed_time(value):
|
|
if not isinstance(value,str):
|
|
return None
|
|
match = re.fullmatch(r'(\d{4})年(\d{1,2})月(\d{1,2})日(\d{1,2})点(\d{1,2})分',value)
|
|
return tuple(map(int,match.groups())) if match else None
|
|
|
|
|
|
async def main(output):
|
|
llm = build_llm_service()
|
|
cases = []
|
|
for delay in (75,135):
|
|
cases.append(dict(name=f'anchored_cross_midnight_delay_{delay}',now=ORIGIN+timedelta(minutes=delay),
|
|
input='没有人受伤。',clocks={'0':ORIGIN.isoformat()},expected=(2026,10,21,23,55)))
|
|
for repeat in range(5):
|
|
cases.append(dict(name=f'legacy_missing_reference_{repeat}',now=ORIGIN+timedelta(hours=2),
|
|
input='没有人受伤。',clocks={},expected=None))
|
|
cases.append(dict(name='explicit_absolute_correction',now=ORIGIN+timedelta(hours=2),
|
|
input='没有人受伤，事故时间更正为2026年10月21日23点40分。',
|
|
clocks={'0':ORIGIN.isoformat()},expected=(2026,10,21,23,40)))
|
|
semaphore=asyncio.Semaphore(4)
|
|
async def run(case):
|
|
calls=[]
|
|
class Recorder:
|
|
async def complete(self,system_prompt,user_message,*,history=None):
|
|
raw=await llm.complete(system_prompt,user_message,history=history)
|
|
calls.append({'kind':'extraction' if system_prompt.startswith('【字段提取】') else 'dialogue',
|
|
'raw':raw})
|
|
return raw
|
|
async with semaphore:
|
|
checks=[]
|
|
try:
|
|
result=await run_collect({'stage_code':'1002','dialogue':HISTORY,
|
|
'dialogue_timestamps':case['clocks'],'input_text':case['input']},
|
|
Recorder(),now=case['now'])
|
|
reply=result['response_text']
|
|
fields=result['accident_fields']
|
|
checks.append(dict(text='continues collection',passed=result['stage_code']=='1002'))
|
|
expected=case['expected']
|
|
if expected is None:
|
|
checks.append(dict(text='no unanchored time in form',passed='accident_time' not in fields))
|
|
checks.append(dict(text='asks for time without inventing it',passed=not re.search(r'\d{4}年.*点.*分',reply)
|
|
and bool(re.search('什么时候|几点|事故.*时间',reply))))
|
|
else:
|
|
checks.append(dict(text='form uses original clock or explicit correction',
|
|
passed=parsed_time(fields.get('accident_time'))==expected))
|
|
value=re.search(r'\d{4}年\d{1,2}月\d{1,2}日\d{1,2}点\d{1,2}分',reply)
|
|
checks.append(dict(text='dialogue confirms same supported value',
|
|
passed=bool(value) and parsed_time(value.group())==expected))
|
|
return dict(name=case['name'],passed=all(c['passed'] for c in checks),checks=checks,
|
|
reply=reply,fields=fields,calls=calls)
|
|
except Exception as exc:
|
|
return dict(name=case['name'],passed=False,error=type(exc).__name__+': '+str(exc),calls=calls)
|
|
results=await asyncio.gather(*(run(c) for c in cases))
|
|
report={'passed':sum(r['passed'] for r in results),'total':len(results),'results':results}
|
|
output.parent.mkdir(parents=True,exist_ok=True)
|
|
output.write_text(json.dumps(report,ensure_ascii=False,indent=2),encoding='utf-8')
|
|
print(json.dumps({k:v for k,v in report.items() if k!='results'}))
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser(__doc__)
|
|
parser.add_argument('--output',type=Path,required=True)
|
|
asyncio.run(main(parser.parse_args().output))
|