wx44wx/ZNJJ-api-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(workflow): align non-motor accident extraction and evaluations

Browse Source
This commit is contained in:
Eric Wang
2026-10-21 15:34:11 +08:00
parent 3fe99e71fd
commit 248eb41c78
56 changed files with 7641 additions and 821 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evals/run.py
+78 -5
View File
@@ -25,6 +25,7 @@ class ChatResult:
stage_codes: list[str]
errors: list[dict]
text: str = ""
form_update: dict = field(default_factory=dict)
@dataclass
@@ -37,6 +38,9 @@ class TranscriptTurn:
errors: list[dict]
ok: bool
error: str | None = None
form_update: dict = field(default_factory=dict)
form: dict = field(default_factory=dict)
expectations: dict = field(default_factory=dict)
@dataclass
@@ -88,6 +92,25 @@ def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]:
f"{path}: step {index} expect.stageCode is required"
)
expect = step["expect"]
for key in ("form", "formMatches"):
if key in expect and not isinstance(expect[key], dict):
raise ValueError(f"{path}: step {index} expect.{key} must be an object")
for pattern in expect.get("formMatches", {}).values():
if not isinstance(pattern, str):
raise ValueError(f"{path}: step {index} formMatches values must be regex strings")
re.compile(pattern)
if "formAbsent" in expect and (
not isinstance(expect["formAbsent"], list)
or not all(isinstance(key, str) for key in expect["formAbsent"])
):
raise ValueError(f"{path}: step {index} expect.formAbsent must be a list of keys")
for key in ("replyMatches", "replyNotMatches"):
if key in expect:
if not isinstance(expect[key], str):
raise ValueError(f"{path}: step {index} expect.{key} must be a regex string")
re.compile(expect[key])
if name_filters and not any(value in name for value in name_filters):
continue
@@ -103,6 +126,7 @@ def parse_sse(response: httpx.Response) -> ChatResult:
stage_codes: list[str] = []
errors: list[dict] = []
text_parts: list[str] = []
form_update: dict = {}
event_name = ""
data_lines: list[str] = []
@@ -116,6 +140,10 @@ def parse_sse(response: httpx.Response) -> ChatResult:
chunk = payload.get("text")
if isinstance(chunk, str) and chunk:
text_parts.append(chunk)
elif event_name == "formUpdate":
if not isinstance(payload, dict):
raise ValueError("formUpdate SSE payload must be an object")
form_update.update(payload)
elif event_name == "error":
errors.append(payload)
event_name = ""
@@ -134,6 +162,7 @@ def parse_sse(response: httpx.Response) -> ChatResult:
stage_codes=stage_codes,
errors=errors,
text="".join(text_parts),
form_update=form_update,
)
@@ -143,6 +172,7 @@ def run_chat_step(
session_id: str,
client_mode: str,
text: str,
need_form_update: bool = False,
) -> ChatResult:
with client.stream(
"POST",
@@ -153,7 +183,7 @@ def run_chat_step(
"timeStamp": timestamp(),
"text": text,
"clientMode": client_mode,
"needFormUpdate": False,
"needFormUpdate": need_form_update,
"useTextChunk": False,
},
) as response:
@@ -194,6 +224,10 @@ def print_turn(turn: TranscriptTurn) -> None:
f" stage: expected={turn.expected_stage_code} "
f"actual={turn.actual_stage_codes}"
)
if turn.expectations.keys() - {"stageCode"}:
print(f" expect: {turn.expectations}")
if turn.form_update or turn.form:
print(f" formUpdate: {turn.form_update}; form: {turn.form}")
if turn.error:
print(f" error: {turn.error}")
if turn.errors:
@@ -224,6 +258,10 @@ def transcript_to_markdown(transcript: CaseTranscript) -> str:
f"- actual stageCodes: `{turn.actual_stage_codes}`",
]
)
if turn.expectations.keys() - {"stageCode"}:
lines.append(f"- expectations: `{json.dumps(turn.expectations, ensure_ascii=False)}`")
lines.append(f"- formUpdate: `{json.dumps(turn.form_update, ensure_ascii=False)}`")
lines.append(f"- cumulative form: `{json.dumps(turn.form, ensure_ascii=False)}`")
if turn.error:
lines.append(f"- error: `{turn.error}`")
if turn.errors:
@@ -256,6 +294,9 @@ def transcript_to_json(transcript: CaseTranscript) -> dict:
"errors": turn.errors,
"ok": turn.ok,
"error": turn.error,
"expectations": turn.expectations,
"formUpdate": turn.form_update,
"form": turn.form,
}
for turn in transcript.turns
],
@@ -280,6 +321,27 @@ def save_transcript(output_dir: Path, transcript: CaseTranscript) -> Path:
return md_path
def check_expectations(result: ChatResult, expect: dict, form: dict) -> list[str]:
failures = []
if result.stage_codes != [expect["stageCode"]]:
failures.append("stageCode mismatch")
if result.errors:
failures.append("SSE errors returned")
for key, value in expect.get("form", {}).items():
if key not in form or type(form[key]) is not type(value) or form[key] != value:
failures.append(f"form.{key}: expected {value!r}, actual {form.get(key)!r}")
for key, pattern in expect.get("formMatches", {}).items():
if not isinstance(form.get(key), str) or not re.search(pattern, form[key]):
failures.append(f"form.{key}: expected /{pattern}/, actual {form.get(key)!r}")
for key in expect.get("formAbsent", []):
if key in form:
failures.append(f"form.{key}: expected absent, actual {form[key]!r}")
for key, should_match in (("replyMatches", True), ("replyNotMatches", False)):
if key in expect and bool(re.search(expect[key], result.text)) != should_match:
failures.append(f"{key}: {expect[key]!r}")
return failures
def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
session_id = session_id_for(case["name"])
transcript = CaseTranscript(
@@ -295,6 +357,11 @@ def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
f"({case['_path']}) session={session_id} ==="
)
form: dict = {}
need_form_update = any(
any(key in step["expect"] for key in ("form", "formAbsent", "formMatches"))
for step in case["steps"]
)
try:
for step_number, step in enumerate(case["steps"], start=1):
expected = step["expect"]["stageCode"]
@@ -305,8 +372,9 @@ def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
session_id=session_id,
client_mode=case["clientMode"],
text=user_input,
need_form_update=need_form_update,
)
except (httpx.HTTPError, json.JSONDecodeError, KeyError) as exc:
except (httpx.HTTPError, ValueError, KeyError) as exc:
turn = TranscriptTurn(
step=step_number,
input=user_input,
@@ -322,7 +390,9 @@ def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
print_turn(turn)
break
ok = result.stage_codes == [expected]
form.update(result.form_update)
failures = check_expectations(result, step["expect"], form)
ok = not failures
turn = TranscriptTurn(
step=step_number,
input=user_input,
@@ -331,7 +401,10 @@ def run_case(client: httpx.Client, case: dict) -> CaseTranscript:
reply=result.text,
errors=result.errors,
ok=ok,
error=None if ok else "stageCode mismatch",
error="; ".join(failures) if failures else None,
form_update=dict(result.form_update),
form=dict(form),
expectations=step["expect"],
)
transcript.turns.append(turn)
print_turn(turn)
@@ -395,7 +468,7 @@ def main() -> int:
args = parse_args()
try:
cases = load_cases(args.cases_dir, args.case)
except (OSError, ValueError, yaml.YAMLError) as exc:
except (OSError, ValueError, yaml.YAMLError, re.error) as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 2