From ddcfe5bd056aeedec2e4f6b1557018b108e10288 Mon Sep 17 00:00:00 2001 From: Eric Wang Date: Wed, 28 Oct 2026 13:51:08 +0800 Subject: [PATCH] feat(evals): add shared case groups and selectable test suites --- evals/README.md | 245 ++++++++++++++++++-------------------- evals/cases/README.md | 56 +++++++++ evals/catalog.py | 93 +++++++++++++++ evals/prompt_compare.py | 53 ++++++--- evals/run.py | 32 +++-- evals/suites.yaml | 48 ++++++++ test/test_eval_catalog.py | 138 +++++++++++++++++++++ 7 files changed, 511 insertions(+), 154 deletions(-) create mode 100644 evals/cases/README.md create mode 100644 evals/catalog.py create mode 100644 evals/suites.yaml create mode 100644 test/test_eval_catalog.py diff --git a/evals/README.md b/evals/README.md index b630fb9..4c2880c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -1,161 +1,146 @@ -# Workflow evaluations +# 评估案例与分组运行 -The evaluation runner calls a live FastAPI service backed by FastGPT. Each -conversation step checks the SSE `stage_code.nextStageCode` value and optional -form / reply expectations. During the run it prints the dialogue and saves -transcripts. Cases cover the current LangGraph backend and workflow snapshots; -import `workflow/20261027` when evaluating the current prompts through FastGPT. -The original case comments retain their historical workflow versions. +`evals/cases/` 当前保留52个完整对话案例、8个目录分组。每个文件独立创建会话，检查阶段码，以及可选的表单、回复断言。 -## Case format +- `python -m evals.run`：通过HTTP调用已经启动的API服务，可测试FastGPT或LangGraph后端。 +- `python -m evals.prompt_compare`：在进程内启动生产文本路由和后端生命周期，使用配置中的真实LLM测试本地LangGraph，不初始化无关语音模型。 -Add YAML files anywhere under `evals/cases/`: +两个入口使用相同的案例目录、筛选参数及断言。所有Python命令使用Conda `py3.11`。 -- `complex_vehicle/`: three-car collisions must immediately return `0003` - with the approved handoff wording in both client modes. Assert the first - explicit three-car turn transfers, extracts count/cause, leaves unknown - injury/time/scene fields absent, and asks no further collection or photo questions. - Semantic probes additionally cover four cars, a known count in an otherwise - empty form, new corrections, uncertain counts, unrelated vehicles, and lower bounds. -- `customer_feedback/`: replay repeated connection checks and the customer's - complete single-car collision description in both client modes. A clearly - described motor vehicle hitting a fixed object, without other vehicle - involvement clues, must extract `vehicle_count: 1` and skip count questions. - Complaint wording must not discard known fields or restart confirmation. - `python -m evals.prompt_probes --runtime --output ...` additionally replays - the old erroneous count question to test recovery after a user challenges it. -- `collection_skip_known_info/`: skip known fields and clarify missing information. - The two files retaining the historical `single_no_inferred_count` name now - follow the user-approved boundary: reversing into a fixed object, without - other vehicle involvement clues, extracts one car. After time confirmation, - route to single-car photography without a separate count answer. Both cases - check `form.vehicle_count: 1` and reject repeated count/type questions. -- `non_motor_vehicle/`: both client modes, with separate flow and extraction cases. - `all_info`, `self_collision`, `no_other_vehicle`, and `explicit_correction` - cover conversation flow and known facts. `electric_bicycle` verifies an - electric bicycle hitting a fixed object and transferring to `0003` without - asking for car counts. These cases accept the described non-motor-vehicle - collision as having `vehicle_count: 0` from the first accident description. - `zero_count_bundle` / `zero_count_history` check zero extraction and retention. - Explicit correction to one car must update the count to `1`. - Both collection and extraction prompts in `workflow/20261021` use this policy - only for an explicitly described non-motor-vehicle single-party collision with - a fixed object; other vehicle involvement or unclear relations do not imply zero. -- `time_confirmation/`: mandatory full-date confirmation, retaining supplemental - information while confirmation is pending, and confirming corrected times. -- `no_false/`: ASR noise must not cause premature transfer. -- `photo_button_guidance/`: button help and failure handling. -- `photo_skip_plate_confirm/`: current photo transitions without plate confirmation. +## 先预览，再运行 -Optional semantic probes support `--exclude-case `; excluded -names are recorded separately and are never counted as passes. For the current -quantity-only iteration, defer the historical time-anchor boundary with: +以下命令只读取案例，不启动应用、不调用模型、不创建结果目录，也不需要配置真实模型： ```bash -python -m evals.prompt_probes --runtime --exclude-case missing_historical_time_anchor --output evals/results/quantity-probes.json +conda run -n py3.11 python -m evals.run --list-groups +conda run -n py3.11 python -m evals.run --list-suites +conda run -n py3.11 python -m evals.run --suite smoke --mode direct --list +conda run -n py3.11 python -m evals.prompt_compare --group complex_vehicle --list ``` -Obsolete scripts requiring repeated time/count confirmations or the removed -`2004` / `2016` photo states have been removed. Photo transition cases use a -complete accident description, confirm its standardized time, then reach their -target branch. Time confirmation is mandatory even when all other information -is already known; an unchanged time is confirmed only once. +`--list`列出案例名、客户端模式和相对路径；列表末尾显示本次选择数量及目录中的总数量。`--list-groups`和`--list-suites`的计数也会应用当前筛选条件。 + +## 目录分组与测试集 + +`--group`精确选择现有目录，不靠名称子串猜测分组： + +| 分组 | 数量 | 主要覆盖 | +| --- | ---: | --- | +| `collection_skip_known_info` | 12 | 已知信息跳过、缺项补充、人伤不确定、单方数量推断 | +| `complex_vehicle` | 2 | 首次明确三车事故即转人工，未知字段不补造 | +| `customer_feedback` | 4 | 客户原始反馈、连续呼叫、已知信息不重复问 | +| `non_motor_vehicle` | 14 | 非机动车参与、零机动车提取、保留已知事实、类型更正 | +| `no_false` | 2 | ASR噪声不误转人工 | +| `time_confirmation` | 2 | 独立确认完整时间、补充信息保留、更正后重新确认 | +| `photo_button_guidance` | 12 | 单双车按钮指引、失败处理、不重复说明 | +| `photo_skip_plate_confirm` | 4 | 当前拍照步骤转换，不额外确认车牌 | + +`--suite`选择跨目录测试集，定义集中在[suites.yaml](suites.yaml)，只引用案例路径，不复制输入或断言： + +| 测试集 | 两种模式合计 | 适用场景 | +| --- | ---: | --- | +| `smoke` | 12 | 修改后的快速回归；配合`--mode direct`仅6个案例 | +| `collection` | 36 | 采集提示词、表单保持、时间与事故分流 | +| `vehicle_count` | 22 | 数量推断、零机动车、类型更正、三车和客户反馈 | +| `safety` | 20 | 复杂事故、非机动车、ASR噪声、不确定人伤 | +| `photo` | 16 | 拍照节点和按钮交互 | +| `feedback` | 6 | 客户反馈，包括三车立即转人工 | +| `full` | 52 | 全量回归；不提供筛选条件也执行全量 | + +详细场景索引及保留历史文件名的原因见[cases/README.md](cases/README.md)。 + +## 局部测试示例 + +先启动待测试API，再运行远程HTTP评估： + +```bash +# 只测试三车分流，先预览再执行 +conda run -n py3.11 python -m evals.run --group complex_vehicle --mode direct --list +conda run -n py3.11 python -m evals.run --group complex_vehicle --mode direct + +# 数量相关案例，仅direct客户端 +conda run -n py3.11 python -m evals.run --suite vehicle_count --mode direct + +# 同时选两个目录 +conda run -n py3.11 python -m evals.run --group time_confirmation --group complex_vehicle + +# 拍照测试集内只选双车，排除按钮失败场景 +conda run -n py3.11 python -m evals.run --suite photo --case dual --exclude-case failure + +# 指定服务及结果目录 +conda run -n py3.11 python -m evals.run --suite smoke --base-url http://127.0.0.1:8000 --output-dir evals/results/smoke +``` + +本地LangGraph评估使用完全相同的筛选方式： + +```bash +conda run -n py3.11 python -m evals.prompt_compare --suite smoke --mode direct --output-dir evals/results/local-smoke +conda run -n py3.11 python -m evals.prompt_compare --suite feedback --output-dir evals/results/local-feedback +conda run -n py3.11 python -m evals.prompt_compare --suite full --workers 4 --output-dir evals/results/local-full +``` + +执行本地评估需要配置`AGENT_BACKEND=langgraph`和真实的`LLM_PROVIDER`；建议用`AGENT_STORAGE=memory`隔离评估会话。`--output-dir`在实际执行时必填，预览时不必填。`metadata.json`记录筛选条件、选中的案例名、所有案例及提示词哈希，便于核对本轮覆盖范围。 + +远程入口继续支持`python evals/run.py`、`EVAL_BASE_URL`、`--timeout`、`--insecure`、`--no-save`；本地入口继续支持`--prompt-dir`提示词快照和`--workers`并发度。两个入口均支持`--cases-dir`指定其他案例目录。 + +## 筛选与去重规则 + +- 同类参数内部取并集：重复`--group`、`--suite`或`--case`表示任意一个匹配。 +- 不同参数之间取交集：例如`--suite photo --mode direct --case dual`只选拍照集中的direct双车案例。 +- `--case`保持原有的案例名子串匹配；`--exclude-case`最后按名称子串排除。 +- 重叠测试集中的同一个文件只执行一次，不因重复指定测试集而复制会话。 +- 未知分组、未知测试集或空选择返回配置错误，避免把“没有跑任何案例”当作成功。 +- 案例名必须唯一，避免输出文件相互覆盖。 + +此次整理未删除、移动或改写任何原案例。核对结果：同一客户端模式内，没有输入和断言完全相同的重复文件，也没有完整输入序列相同的文件。direct和browser_addon的相似案例验证不同请求模式与阶段码，保留为独立覆盖；只想测一种模式时使用`--mode`减少执行量。共同的欢迎和时间确认轮次是对话上下文的一部分，不单独抽掉。 + +## 新增案例与断言 + +把YAML放入现有目录，或新建业务目录；目录自动成为`--group`。新案例会自动纳入全量执行；按需要更新`suites.yaml`中的跨目录路径规则。套件定义放在cases目录外，避免被当成对话案例读取。 ```yaml -name: direct_initial_stage +name: direct_example clientMode: direct - steps: - input: 新对话 expect: stageCode: "1001" - - input: 继续办理 expect: stageCode: "1002" ``` -`clientMode` in the case file is the API request setting (`direct` or -`browser_addon`). Steps in one file share the same generated `sessionId`. -Different cases always use different session IDs. +`clientMode`为`direct`或`browser_addon`，后者的阶段断言需要`browser_addon.`前缀。同一文件的各轮共享一个新生成的sessionId，不同文件使用不同sessionId。每个案例在首个失败轮次停止，并尝试清理会话。 -Optional `expect` fields: +可选断言： -- `form`: partial equality against the cumulative form, including value types. -- `formMatches`: field-to-regex mapping for free-text descriptions. -- `formAbsent`: fields that must not have been extracted yet. -- `replyMatches` / `replyNotMatches`: required / forbidden reply regexes. +| 字段 | 含义 | +| --- | --- | +| `form` | 累计表单的部分精确相等，包含类型检查 | +| `formMatches` | 自由文本字段的正则检查 | +| `formAbsent` | 尚不得提取的字段 | +| `replyMatches` | 回复必须匹配的正则 | +| `replyNotMatches` | 回复禁止匹配的正则 | -If any step has a form expectation, the runner sends `needFormUpdate: true` -throughout that case, merges SSE `formUpdate` deltas across turns, and records -both updates and form snapshots. Thus an omitted unchanged field is accepted, -while a later incorrect overwrite fails. SSE errors also fail the step. -Existing cases without reply expectations still require manual review of their -commented wording requirements. +一个案例包含任意表单断言时，执行器全程请求`needFormUpdate: true`，跨轮合并SSE表单增量。因此未重复返回不变字段可以接受，后续错误覆盖会失败。SSE错误也使本轮失败。仅有注释、没有回复断言的旧案例仍需要查看实际对话。 -```yaml - - input: 没有人员伤亡。 - expect: - stageCode: "1002" - form: {has_injury: false, has_non_motor_vehicle_hit: true} - replyMatches: "(什么时候|几点|事故.*时间)" - replyNotMatches: "(有没有|是否).*非机动车" -``` +当前边界：完整倒车撞固定物且无其他车辆参与线索按1辆处理；已知至少3辆机动车参与立即转人工；单/双车进入拍照前需要独立确认时间。非机动车参与的既有分支保留必要采集及时间确认。通过FastGPT评估当前提示词时，应先导入本仓库`workflow/20261027`快照；案例中的历史版本注释保留供追溯。 -## Run +## 结果与补充语义检查 -Start the API, then execute: +运行时打印每轮输入、回复和断言结果。远程入口默认保存到`evals/results//`；本地入口保存到指定目录： + +- `*.md`：可读对话记录。 +- `*.json`：每轮阶段码、回复、表单增量、累计表单和断言。 +- 本地入口另保存`metadata.json`、`summary.json`及运行日志。 + +两个入口实际执行时，任一案例失败返回1，配置错误返回2，通过返回0。`--list`等预览成功返回0。 + +`prompt_probes.py`用于局部状态快照和历史回放，独立于cases；`time_boundary.py`检查历史时间来源边界。它们不读取`suites.yaml`。示例： ```bash -python evals/run.py +conda run -n py3.11 python -m evals.prompt_probes --runtime --case complex_ --output evals/results/complex-probes.json +conda run -n py3.11 python -m evals.prompt_probes --runtime --exclude-case missing_historical_time_anchor --output evals/results/probes.json ``` -Use another API address: - -```bash -python evals/run.py --base-url http://192.168.1.10:8000 -``` - -Run selected cases: - -```bash -python evals/run.py --case direct -python evals/run.py --case browser_addon -python evals/run.py --case non_motor_vehicle -python evals/run.py --case non_motor_vehicle_electric_bicycle -python evals/run.py --case non_motor_vehicle_zero_count -python evals/run.py --case time_confirmation -python evals/run.py --case photo_skip_plate_confirm -``` - -There are 14 non-motor-vehicle cases: 10 flow cases and 4 zero-count extraction -cases. Both groups require `vehicle_count: 0` for the described non-motor-vehicle -collisions, while unrelated chatter must leave accident fields absent. Vehicle -type corrections, time confirmation, and transfer checks remain active. The -runner still stops each case at its first failure. - -## Dialogue and transcripts - -While a case runs, each step prints the user input, bot reply (`text_delta`), -and stage-code check result. - -By default, after the run finishes, transcripts are written under -`evals/results//`: - -- `*.md` — readable conversation log -- `*.json` — structured step results - -Save to a specific directory: - -```bash -python evals/run.py --output-dir evals/results/manual-run --case direct -``` - -Print live dialogue only, without writing files: - -```bash -python evals/run.py --no-save --case direct -``` - -The process exits with code `1` when an expectation fails and code `2` when -case configuration is invalid. +`missing_historical_time_anchor`是此前用户要求暂缓的历史时间基准检查；仅显式排除时跳过，排除项单独记录，不能计为通过。本次整理没有改变这个边界。 diff --git a/evals/cases/README.md b/evals/cases/README.md new file mode 100644 index 0000000..187c785 --- /dev/null +++ b/evals/cases/README.md @@ -0,0 +1,56 @@ +# 对话案例索引 + +本目录保留52个YAML案例，输入、断言和文件路径均保持原样。执行入口、分组参数及断言格式见[评估说明](../README.md)。跨目录测试集定义在[../suites.yaml](../suites.yaml)，不会复制这里的案例。 + +## 信息采集 + +| 目录 | 文件名中的场景 | 检查重点 | +| --- | --- | --- | +| `collection_skip_known_info` | `single_all_info` / `dual_all_info` | 一次提供所有信息，只独立确认时间，然后分流 | +| 同上 | `dual_fill_missing` / `dual_bundle_remaining` | 分轮补充、批量补充后保留已知字段 | +| 同上 | `dual_unclear_injury` | 人伤不确定先澄清，再使用已有信息 | +| 同上 | `single_no_inferred_count` | 历史文件名；最新边界已改为倒车撞固定物提取1，不再问数量 | +| `time_confirmation` | `correction` | 首次确认、补充信息、时间更正和再次确认 | +| `customer_feedback` | `repeated_question` | 客户原始多轮反馈，单车描述提取1，后续不重复问数量 | +| 同上 | `known_fields` | 时间确认与抱怨一起表达时，仍保留已知信息 | + +## 分流与提取边界 + +| 目录 | 文件名中的场景 | 检查重点 | +| --- | --- | --- | +| `complex_vehicle` | `three_immediate` | 首次明确三车追尾立即0003，精确话术及已知字段提取 | +| `non_motor_vehicle` | `all_info` | 信息完整的非机动车事故分流 | +| 同上 | `self_collision` / `electric_bicycle` | 分轮说明单方碰撞及具体电动自行车表达 | +| 同上 | `no_other_vehicle` | “没有其他车”不否定自身非机动车参与 | +| 同上 | `explicit_correction` | 明确更正车辆类型时更新相关字段 | +| 同上 | `zero_count_bundle` / `zero_count_history` | 机动车数量0的提取与跨轮保留 | +| `no_false` | `asr_no_false_0002` / `asr_no_false_0003` | 不同ASR噪声序列不误判拒绝或人伤转人工 | + +## 拍照 + +| 目录 | 文件名中的场景 | 检查重点 | +| --- | --- | --- | +| `photo_button_guidance` | `single_guidance` / `dual_guidance` | 拍照按钮使用说明 | +| 同上 | `single_failure` / `dual_failure` | 拍照失败与客户端事件处理 | +| 同上 | `single_no_repeat` / `dual_no_repeat` | 已解释后不重复完整说明 | +| `photo_skip_plate_confirm` | `single` / `dual` | 现有拍照状态转换，不额外确认车牌 | + +除两份ASR案例仅覆盖direct外，其余场景各有direct和browser_addon两个文件；它们验证不同客户端模式和阶段码前缀，不视为重复文件。相似场景的用户表达、历史路径或字段断言有区别，不因标题相近合并或删除。 + +## 常用选择 + +```bash +# 快速回归，6个direct案例 +conda run -n py3.11 python -m evals.run --suite smoke --mode direct --list + +# 只看三车事故 +conda run -n py3.11 python -m evals.run --group complex_vehicle --list + +# 只看非机动车零数量保留 +conda run -n py3.11 python -m evals.run --group non_motor_vehicle --case zero_count --list + +# 组合拍照目录，但只执行一次每个匹配文件 +conda run -n py3.11 python -m evals.run --suite photo --group photo_button_guidance --case dual --list +``` + +以上命令在仓库根目录执行。去掉`--list`后运行实际API评估；本地LangGraph入口`python -m evals.prompt_compare`支持相同筛选参数，执行时还需`--output-dir`。 diff --git a/evals/catalog.py b/evals/catalog.py new file mode 100644 index 0000000..4fcfde0 --- /dev/null +++ b/evals/catalog.py @@ -0,0 +1,93 @@ +"""Shared case selection for remote API and local LangGraph evaluations.""" + +import argparse +from collections import Counter +from pathlib import Path + +import yaml + +SUITES_PATH = Path(__file__).with_name("suites.yaml") + + +def load_suites(path: Path = SUITES_PATH) -> dict[str, dict]: + suites = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(suites, dict): + raise ValueError(f"{path}: suites must be an object") + for name, suite in suites.items(): + if not isinstance(name, str) or not isinstance(suite, dict): + raise ValueError(f"{path}: invalid suite {name!r}") + patterns = suite.get("include") + if not isinstance(patterns, list) or not patterns or not all( + isinstance(pattern, str) and pattern for pattern in patterns + ): + raise ValueError(f"{path}: suite {name!r} needs nonempty include patterns") + return suites + + +def add_selection_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--case", action="append", default=[], + help="Name substring; repeat to match any") + parser.add_argument("--group", action="append", default=[], + help="Exact case directory group; repeat to match any") + parser.add_argument("--suite", action="append", default=[], + help="Named suite from evals/suites.yaml; repeat for union") + parser.add_argument("--mode", choices=("direct", "browser_addon"), + help="Only this client mode") + parser.add_argument("--exclude-case", action="append", default=[], + help="Exclude names containing this value; repeatable") + listing = parser.add_mutually_exclusive_group() + listing.add_argument("--list", action="store_true", dest="list_cases", + help="List selected cases without API/model calls") + listing.add_argument("--list-groups", action="store_true", + help="List directory groups and counts without running") + listing.add_argument("--list-suites", action="store_true", + help="List named suites and counts without running") + + +def select_cases( + cases: list[dict], *, names=(), groups=(), suites=(), mode=None, + exclude_names=(), suite_definitions: dict | None = None, +) -> list[dict]: + """Union within each filter; intersection between filters; exclusions last. + + Operate on the loaded catalog once, so overlapping suites never duplicate + runs. Client modes and different assertions remain independent coverage. + """ + available_groups = {case["_group"] for case in cases} + unknown = set(groups) - available_groups + if unknown: + raise ValueError(f"Unknown groups: {sorted(unknown)}; available: {sorted(available_groups)}") + definitions = load_suites() if suite_definitions is None else suite_definitions + unknown = set(suites) - definitions.keys() + if unknown: + raise ValueError(f"Unknown suites: {sorted(unknown)}; available: {sorted(definitions)}") + patterns = [pattern for name in suites for pattern in definitions[name]["include"]] + selected = [case for case in cases + if (not names or any(value in case["name"] for value in names)) + and (not groups or case["_group"] in groups) + and (not patterns or any(Path(case["_relative_path"]).match(pattern) for pattern in patterns)) + and (mode is None or case["clientMode"] == mode) + and not any(value in case["name"] for value in exclude_names)] + if not selected: + raise ValueError("No evaluation cases match the selected filters") + return selected + + +def listing_requested(args) -> bool: + return args.list_cases or args.list_groups or args.list_suites + + +def print_case_listing(args, catalog: list[dict], selected: list[dict]) -> None: + if args.list_groups: + counts = Counter(case["_group"] for case in selected) + for group, count in sorted(counts.items()): + print(f"{group}\t{count}") + elif args.list_suites: + for name, suite in load_suites().items(): + count = sum(any(Path(case["_relative_path"]).match(pattern) + for pattern in suite["include"]) for case in selected) + print(f"{name}\t{count}\t{suite.get('description', '')}") + else: + for case in selected: + print(f"{case['name']}\t{case['clientMode']}\t{case['_relative_path']}") + print(f"Selected {len(selected)} of {len(catalog)} cases") diff --git a/evals/prompt_compare.py b/evals/prompt_compare.py index 930b17d..c31c8e3 100644 --- a/evals/prompt_compare.py +++ b/evals/prompt_compare.py @@ -9,29 +9,47 @@ from contextlib import redirect_stdout from hashlib import sha256 import json from pathlib import Path +import re import subprocess import time -from fastapi import FastAPI -from fastapi.testclient import TestClient -from src.core.config import Config -from src.api.endpoints import router -from src.core.fastgpt_client import lifespan -from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript +import yaml -# Exercise the production text router and backend lifespan, without loading -# unrelated voice models or downloading their assets during text evaluations. -app = FastAPI(lifespan=lifespan) -app.include_router(router) +from evals.catalog import add_selection_arguments, listing_requested, print_case_listing, select_cases +from evals.run import DEFAULT_CASES_DIR, load_cases, run_case, save_transcript def main(): parser = argparse.ArgumentParser(__doc__) - parser.add_argument('--output-dir', type=Path, required=True) + parser.add_argument('--output-dir', type=Path) parser.add_argument('--workers', type=int, default=4) - parser.add_argument('--case', action='append', default=[]) + parser.add_argument('--cases-dir', type=Path, default=DEFAULT_CASES_DIR) + add_selection_arguments(parser) parser.add_argument('--prompt-dir', type=Path, help='Optional prompt snapshot; other prompts use the project version') args = parser.parse_args() + try: + catalog = load_cases(args.cases_dir, []) + cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite, + mode=args.mode, exclude_names=args.exclude_case) + except (OSError, ValueError, yaml.YAMLError, re.error) as exc: + parser.error(str(exc)) + if listing_requested(args): + print_case_listing(args, catalog, cases) + return 0 + if args.output_dir is None: + parser.error('--output-dir is required when running evaluations') + if args.workers < 1: + parser.error('--workers must be at least 1') + + # Previewing the catalog never imports or starts the application/model client. + from fastapi import FastAPI + from fastapi.testclient import TestClient + from src.core.config import Config + from src.api.endpoints import router + from src.core.fastgpt_client import lifespan + + app = FastAPI(lifespan=lifespan) + app.include_router(router) if args.prompt_dir: import src.agent.prompts as registry original_loader = registry.load_prompt @@ -42,14 +60,16 @@ def main(): if Config.AGENT_BACKEND != 'langgraph' or not Config.LLM_PROVIDER or Config.LLM_PROVIDER == 'fake': raise SystemExit('Requires langgraph backend and real openai provider') args.output_dir.mkdir(parents=True, exist_ok=True) - cases = load_cases(DEFAULT_CASES_DIR, args.case) from src.agent.prompts import load_prompt metadata = { **{k: v for k, v in (Config.llm_profile(Config.LLM_PROVIDER) or {}).items() if k in ('model', 'temperature')}, 'backend': Config.AGENT_BACKEND, 'workers': args.workers, 'commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], text=True).strip(), - 'case_hashes': {str(p.relative_to(DEFAULT_CASES_DIR)): sha256(p.read_bytes()).hexdigest() - for p in sorted(DEFAULT_CASES_DIR.rglob('*.yaml'))}, + 'selection': {'case': args.case, 'group': args.group, 'suite': args.suite, + 'mode': args.mode, 'exclude_case': args.exclude_case}, + 'selected_cases': [case['name'] for case in cases], + 'case_hashes': {str(p.relative_to(args.cases_dir)): sha256(p.read_bytes()).hexdigest() + for p in sorted(args.cases_dir.rglob('*.yaml'))}, 'prompt_hashes': {p.name: sha256(load_prompt(p.stem).encode('utf-8')).hexdigest() for p in Path('src/agent/prompts').glob('*.md')}, } (args.output_dir / 'metadata.json').write_text(json.dumps(metadata, indent=2), encoding='utf-8') @@ -79,7 +99,8 @@ def main(): json.dumps(summary, ensure_ascii=False, indent=2), encoding='utf-8') log.flush() print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 if summary['passed'] == summary['total'] else 1 if __name__ == '__main__': - main() + raise SystemExit(main()) diff --git a/evals/run.py b/evals/run.py index ab276a8..0491c6b 100644 --- a/evals/run.py +++ b/evals/run.py @@ -114,11 +114,17 @@ def load_cases(cases_dir: Path, name_filters: list[str]) -> list[dict]: if name_filters and not any(value in name for value in name_filters): continue - case["_path"] = str(path.relative_to(EVALS_DIR)) + relative_path = path.relative_to(cases_dir) + case["_relative_path"] = relative_path.as_posix() + case["_group"] = relative_path.parent.as_posix() + case["_path"] = str(path.relative_to(EVALS_DIR)) if path.is_relative_to(EVALS_DIR) else str(path) cases.append(case) if not cases: raise ValueError(f"No evaluation cases found under {cases_dir}") + names = [case["name"] for case in cases] + if len(set(names)) != len(names): + raise ValueError("Evaluation case names must be unique") return cases @@ -435,12 +441,12 @@ def parse_args() -> argparse.Namespace: default=DEFAULT_CASES_DIR, help="Directory containing YAML cases", ) - parser.add_argument( - "--case", - action="append", - default=[], - help="Run cases whose names contain this value; repeatable", - ) + # Support both `python evals/run.py` and `python -m evals.run`. + try: + from .catalog import add_selection_arguments + except ImportError: + from catalog import add_selection_arguments + add_selection_arguments(parser) parser.add_argument( "--output-dir", type=Path, @@ -467,11 +473,21 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() try: - cases = load_cases(args.cases_dir, args.case) + from .catalog import listing_requested, print_case_listing, select_cases + except ImportError: + from catalog import listing_requested, print_case_listing, select_cases + try: + catalog = load_cases(args.cases_dir, []) + cases = select_cases(catalog, names=args.case, groups=args.group, suites=args.suite, + mode=args.mode, exclude_names=args.exclude_case) except (OSError, ValueError, yaml.YAMLError, re.error) as exc: print(f"ERROR: {exc}", file=sys.stderr) return 2 + if listing_requested(args): + print_case_listing(args, catalog, cases) + return 0 + output_dir: Path | None = None if not args.no_save: output_dir = args.output_dir or (DEFAULT_OUTPUT_DIR / local_run_stamp()) diff --git a/evals/suites.yaml b/evals/suites.yaml new file mode 100644 index 0000000..cd6f251 --- /dev/null +++ b/evals/suites.yaml @@ -0,0 +1,48 @@ +# Paths are relative to cases/. Suites select files; no inputs or assertions +# are copied here. All suites run both client modes unless --mode is supplied. +smoke: + description: 快速覆盖单车、双车、复杂事故、非机动车、时间更正和拍照转换 + include: + - collection_skip_known_info/*_single_all_info.yaml + - collection_skip_known_info/*_dual_all_info.yaml + - complex_vehicle/*.yaml + - non_motor_vehicle/*_electric_bicycle.yaml + - time_confirmation/*.yaml + - photo_skip_plate_confirm/*_single.yaml +collection: + description: 信息采集、表单保持、时间确认、非机动车和复杂事故分流 + include: + - collection_skip_known_info/*.yaml + - customer_feedback/*.yaml + - complex_vehicle/*.yaml + - non_motor_vehicle/*.yaml + - time_confirmation/*.yaml + - no_false/*.yaml +vehicle_count: + description: 单方数量推断、零机动车提取、类型更正、三车转人工及客户反馈 + include: + - collection_skip_known_info/*_single_no_inferred_count.yaml + - non_motor_vehicle/*.yaml + - complex_vehicle/*.yaml + - customer_feedback/*.yaml +safety: + description: 复杂事故立即转人工、非机动车分流、ASR噪声和不确定人伤 + include: + - complex_vehicle/*.yaml + - non_motor_vehicle/*.yaml + - no_false/*.yaml + - collection_skip_known_info/*_dual_unclear_injury.yaml +photo: + description: 单双车拍照转换、按钮指引、失败及避免重复说明 + include: + - photo_button_guidance/*.yaml + - photo_skip_plate_confirm/*.yaml +feedback: + description: 客户反馈回归，包括已知信息不重复问和三车立即转人工 + include: + - customer_feedback/*.yaml + - complex_vehicle/*.yaml +full: + description: 全部案例，等价于不指定测试集 + include: + - "*.yaml" diff --git a/test/test_eval_catalog.py b/test/test_eval_catalog.py new file mode 100644 index 0000000..3c3b438 --- /dev/null +++ b/test/test_eval_catalog.py @@ -0,0 +1,138 @@ +"""Selection tests use real case files and make no API/model calls.""" + +from collections import Counter +from contextlib import nullcontext +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest +import yaml + +from evals.catalog import load_suites, select_cases +from evals.run import CaseTranscript, DEFAULT_CASES_DIR, load_cases + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def catalog(): + return load_cases(DEFAULT_CASES_DIR, []) + + +def test_full_keeps_all_cases_and_modes(catalog): + selected = select_cases(catalog, suites=["full"]) + assert selected == catalog + assert {c["clientMode"] for c in selected} == {"direct", "browser_addon"} + + +def test_overlapping_suites_execute_each_case_once(catalog): + selected = select_cases(catalog, suites=["feedback", "collection", "feedback"]) + expected = set(c["name"] for c in select_cases(catalog, suites=["collection"])) + assert len(selected) == len(expected) + assert {c["name"] for c in selected} == expected + + +def test_filter_intersection_and_exclusion(catalog): + selected = select_cases(catalog, suites=["collection"], + groups=["collection_skip_known_info"], mode="direct", + names=["single", "dual"], exclude_names=["single", "unclear"]) + assert selected + assert all(c["clientMode"] == "direct" and "dual" in c["name"] + and "unclear" not in c["name"] for c in selected) + assert all(c["_group"] == "collection_skip_known_info" for c in selected) + + +@pytest.mark.parametrize("kwargs", [ + {"groups": ["not-a-group"]}, + {"suites": ["not-a-suite"]}, + {"names": ["not-a-case"]}, + {"groups": ["complex_vehicle"], "suites": ["photo"]}, +]) +def test_bad_or_empty_selection_fails(catalog, kwargs): + with pytest.raises(ValueError): + select_cases(catalog, **kwargs) + + +def test_suites_have_live_patterns_and_modes(catalog): + for name, suite in load_suites().items(): + assert {c["clientMode"] for c in select_cases(catalog, suites=[name])} == { + "direct", "browser_addon", + } + for pattern in suite["include"]: + assert any(Path(c["_relative_path"]).match(pattern) for c in catalog), (name, pattern) + assert len(select_cases(catalog, suites=["smoke"])) < len(catalog) + + +def test_directory_group_and_relative_path(catalog): + for case in catalog: + assert case["_group"] == Path(case["_relative_path"]).parent.as_posix() + assert "\\" not in case["_relative_path"] + + +def test_custom_directory_and_duplicate_names(tmp_path): + case = {"name": "custom", "clientMode": "direct", "steps": [ + {"input": "开始", "expect": {"stageCode": "1001"}}, + ]} + group = tmp_path / "custom_group" + group.mkdir() + (group / "a.yaml").write_text(yaml.safe_dump(case), encoding="utf-8") + selected = select_cases(load_cases(tmp_path, []), groups=["custom_group"]) + assert selected[0]["_relative_path"] == "custom_group/a.yaml" + (group / "b.yaml").write_text(yaml.safe_dump(case), encoding="utf-8") + with pytest.raises(ValueError, match="unique"): + load_cases(tmp_path, []) + + +@pytest.mark.parametrize("command", [ + ["-m", "evals.run"], + ["evals/run.py"], + ["-m", "evals.prompt_compare"], +]) +def test_cli_preview_has_no_provider_or_output_requirement(command, tmp_path): + env = {**os.environ, "PYTHONIOENCODING": "utf-8", "LLM_PROVIDER": "fake", + "AGENT_BACKEND": "fastgpt"} + result = subprocess.run( + [sys.executable, *command, "--suite", "feedback", "--group", "complex_vehicle", + "--mode", "direct", "--list"], cwd=ROOT, env=env, + capture_output=True, text=True, encoding="utf-8", timeout=30, + ) + assert result.returncode == 0, result.stderr + assert "direct_complex_three_immediate" in result.stdout + assert "browser_addon_complex_three_immediate" not in result.stdout + assert "Selected 1 of" in result.stdout + + +def test_no_identical_inputs_and_assertions_within_a_mode(catalog): + fingerprints = Counter(json.dumps({"mode": case["clientMode"], "steps": case["steps"]}, + sort_keys=True, ensure_ascii=False) for case in catalog) + assert all(count == 1 for count in fingerprints.values()) + + +@pytest.mark.parametrize("passed,exit_code", [(True, 0), (False, 1)]) +def test_execution_uses_the_preview_selection(monkeypatch, passed, exit_code): + from evals import run + + calls = [] + monkeypatch.setattr(sys, "argv", ["evals.run", "--suite", "feedback", "--suite", "collection", + "--group", "complex_vehicle", "--mode", "direct", "--no-save"]) + monkeypatch.setattr(run.httpx, "Client", lambda **kwargs: nullcontext(None)) + + def fake_run(client, case): + calls.append(case["name"]) + return CaseTranscript(case["name"], case["_path"], case["clientMode"], "test", passed) + + monkeypatch.setattr(run, "run_case", fake_run) + assert run.main() == exit_code + assert calls == ["direct_complex_three_immediate"] + + +def test_cli_unknown_group_is_configuration_error(): + result = subprocess.run( + [sys.executable, "-m", "evals.run", "--group", "unknown", "--list"], + cwd=ROOT, capture_output=True, text=True, encoding="utf-8", timeout=30, + ) + assert result.returncode == 2 + assert "Unknown groups" in result.stderr