Eric Wang
74db6be385
- merge main: workflow/20261021 (collection prompt adds step-5 mandatory time readback-confirm before routing; non-motor/0003 branch rework; accident_cause unified schema), updated eval suites (photo_skip/ button_guidance rewritten, no_false trimmed to asr, new non_motor_vehicle + time_confirmation with form/reply assertions), runner gains form/formMatches/replyMatches expectations - prompts/collection.md regenerated from 20261021; extraction prompt archived (contentExtract description, 6-key whitelist) - collect: extraction call per turn (extraction.md + current time/form injected), merge follows 更新表单 semantics (meaningful values only, number/string equality, diff -> turn_patch); accident_fields now uses external form keys - backend: form_update flows to ChatResult and SSE formUpdate event (stream emits prefix, FormUpdate, body) - fields: FORM_FIELDS whitelist with strict type validation - verified: 155 unit tests; golden evals on real model - photo_skip 4/4, no_false 2/2, photo_button_guidance 12/12, non_motor_vehicle 14/14, collection_skip 10/12, time_confirmation 1/2; remaining 3 failures are model sequence/literal-template adherence, recorded in plan appendix
105 lines
4.3 KiB
Python
105 lines
4.3 KiB
Python
"""20261020 工作流的提示词移植与模型决策回归。"""
|
|
|
|
from datetime import datetime, timezone
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from src.agent.collect import CollectReplyError, run_collect
|
|
from src.agent.llm import FakeLLMService
|
|
from src.agent.prompts import COLLECTION, load_prompt, render_prompt
|
|
|
|
WORKFLOW = Path(__file__).resolve().parents[2] / "workflow/20261021/事故信息采集2025118.json"
|
|
NODES = {
|
|
"collection": "事故信息收集",
|
|
"single_photo": "单车拍照",
|
|
"dual_photo": "双车拍照",
|
|
"single_photo_addon": "单车拍照#6",
|
|
"dual_photo_addon": "双车拍照#7",
|
|
"single_confirm": "单车信息确认",
|
|
"dual_confirm": "双车信息确认",
|
|
"extraction": "文本内容提取事故信息",
|
|
}
|
|
_INPUT_KEYS = {"extraction": "description"}
|
|
|
|
|
|
@pytest.mark.parametrize(("prompt", "node_name"), NODES.items())
|
|
def test_archived_prompt_matches_workflow(prompt, node_name):
|
|
workflow = json.loads(WORKFLOW.read_text(encoding="utf-8-sig"))
|
|
node = next(n for n in workflow["nodes"] if n["name"] == node_name)
|
|
input_key = _INPUT_KEYS.get(prompt, "systemPrompt")
|
|
original = next(i["value"] for i in node["inputs"] if i["key"] == input_key)
|
|
archived = load_prompt(prompt).split("--- 原文开始 ---", 1)[1].split("--- 原文结束 ---", 1)[0]
|
|
assert archived.strip() == original.strip()
|
|
|
|
|
|
def test_render_replaces_time_and_person_variables():
|
|
now = datetime(2026, 10, 20, 12, 0, tzinfo=timezone.utc)
|
|
rendered = render_prompt(COLLECTION, {}, now=now)
|
|
assert "2026-10-20 12:00:00 +0000" in rendered
|
|
rendered = render_prompt("dual_confirm", {
|
|
"person_fields": {"hphm1": "测试A", "hphm2": "测试B", "sfzmwh1": "1234"}
|
|
})
|
|
assert "测试A" in rendered and "测试B" in rendered
|
|
assert "1234" in rendered
|
|
assert "{{$" not in rendered
|
|
assert "未提供" in rendered
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_short_answer_does_not_override_model_routing():
|
|
# “在”可能是明确回答；运行时不得按字数一律拒绝分流。
|
|
llm = FakeLLMService(lambda s, u: "<state>2000</state>信息已齐，请拍摄车辆前方。")
|
|
result = await run_collect({
|
|
"stage_code": "1002", "input_text": "在", "clarify_count": 0,
|
|
"dialogue": [{"role": "assistant", "content": "您还在事故现场吗？"}],
|
|
}, llm)
|
|
assert result["stage_code"] == "2000"
|
|
assert result["clarify_count"] == 0
|
|
assert len(llm.calls) == 2 # 对话 + 提取
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_format_and_stage_errors_share_one_repair_budget():
|
|
replies = iter(["缺少前缀", "<state>9999</state>非法阶段"])
|
|
llm = FakeLLMService(lambda s, u: next(replies))
|
|
with pytest.raises(CollectReplyError):
|
|
await run_collect({"stage_code": "1002", "input_text": "追尾了"}, llm)
|
|
assert len(llm.calls) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_collect_sends_history_as_messages():
|
|
history = [
|
|
{"role": "user", "content": "没有人受伤"},
|
|
{"role": "assistant", "content": "<state>1002</state>请问有几辆车？"},
|
|
]
|
|
llm = FakeLLMService(lambda s, u: "<state>1002</state>请问您还在现场吗？")
|
|
await run_collect({
|
|
"stage_code": "1002", "input_text": "两辆", "dialogue": history,
|
|
}, llm)
|
|
assert llm.calls[0]["history"] == history
|
|
assert llm.calls[0]["user_message"] == "两辆"
|
|
assert "没有人受伤" not in llm.calls[0]["user_message"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_openai_adapter_preserves_message_roles():
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
from src.agent.llm import OpenAILLMService
|
|
|
|
create = AsyncMock(return_value=SimpleNamespace(choices=[
|
|
SimpleNamespace(message=SimpleNamespace(content="<state>1002</state>收到。"))
|
|
]))
|
|
llm = OpenAILLMService(model="test", api_key="test", max_retries=0)
|
|
llm._client = SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create)))
|
|
history = [{"role": "assistant", "content": "之前的问题"}]
|
|
await llm.complete("系统提示词", "本轮输入", history=history)
|
|
assert create.call_args.kwargs["messages"] == [
|
|
{"role": "system", "content": "系统提示词"},
|
|
{"role": "assistant", "content": "之前的问题"},
|
|
{"role": "user", "content": "本轮输入"},
|
|
]
|