feat: add persisted batch text testing

2026-08-10 13:49:24 +08:00
parent 5b8b9fd097
commit 19e8c8c108
27 changed files with 3855 additions and 1450 deletions
@@ -0,0 +1,369 @@
+"""Persistent test suite and fixed-text test case CRUD."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from db.models import TestCase, TestRunCase, TestSuite
+from db.session import get_session
+from fastapi import APIRouter, Depends, HTTPException, Query
+from services.auth import require_admin
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from test_schemas import (
+    TestCaseBulkDeleteIn,
+    TestCaseDefinition,
+    TestCaseOrderIn,
+    TestCaseOut,
+    TestCaseWrite,
+    TestSuiteCreate,
+    TestSuiteOut,
+    TestSuiteUpdate,
+)
+
+
+router = APIRouter(
+    tags=["tests"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+def _new_id(prefix: str) -> str:
+    return f"{prefix}_{uuid4().hex}"
+
+
+async def _touch_suite(session: AsyncSession, suite_id: str) -> None:
+    suite = await session.get(TestSuite, suite_id)
+    if suite is not None:
+        suite.updated_at = datetime.now(UTC)
+
+
+async def _suite_out(session: AsyncSession, row: TestSuite) -> TestSuiteOut:
+    total = await session.scalar(
+        select(func.count()).select_from(TestCase).where(TestCase.suite_id == row.id)
+    )
+    run = await session.scalar(
+        select(func.count(func.distinct(TestRunCase.run_id))).where(
+            TestRunCase.suite_id == row.id
+        )
+    )
+    passed = await session.scalar(
+        select(func.count())
+        .select_from(TestCase)
+        .where(TestCase.suite_id == row.id, TestCase.last_result == "pass")
+    )
+    return TestSuiteOut(
+        id=row.id,
+        name=row.name,
+        description=row.description,
+        case_count=int(total or 0),
+        passed_count=int(passed or 0),
+        run_count=int(run or 0),
+        updated_at=row.updated_at,
+    )
+
+
+def _case_out(row: TestCase) -> TestCaseOut:
+    definition = TestCaseDefinition.model_validate(row.definition or {})
+    return TestCaseOut(
+        id=row.id,
+        suite_id=row.suite_id,
+        name=row.name,
+        description=row.description,
+        input_mode=row.input_mode,
+        last_result=row.last_result,
+        sort_order=row.sort_order,
+        updated_at=row.updated_at,
+        **definition.model_dump(),
+    )
+
+
+def _apply_case_write(row: TestCase, body: TestCaseWrite) -> None:
+    row.name = body.name.strip()
+    row.description = body.description.strip()
+    row.input_mode = body.input_mode
+    row.definition = body.definition().model_dump(mode="json", by_alias=True)
+
+
+async def _renumber_cases(session: AsyncSession, suite_id: str) -> None:
+    rows = (
+        await session.execute(
+            select(TestCase)
+            .where(TestCase.suite_id == suite_id)
+            .order_by(TestCase.sort_order, TestCase.created_at, TestCase.id)
+        )
+    ).scalars().all()
+    offset = len(rows) + 1
+    for index, row in enumerate(rows):
+        row.sort_order = offset + index
+    await session.flush()
+    for index, row in enumerate(rows):
+        row.sort_order = index
+
+
+@router.get("/api/test-suites", response_model=list[TestSuiteOut])
+async def list_test_suites(session: AsyncSession = Depends(get_session)):
+    rows = (
+        await session.execute(select(TestSuite).order_by(TestSuite.updated_at.desc()))
+    ).scalars().all()
+    return [await _suite_out(session, row) for row in rows]
+
+
+@router.post("/api/test-suites", response_model=TestSuiteOut)
+async def create_test_suite(
+    body: TestSuiteCreate,
+    session: AsyncSession = Depends(get_session),
+):
+    row = TestSuite(
+        id=_new_id("suite"),
+        name=body.name.strip(),
+        description=body.description.strip(),
+    )
+    session.add(row)
+    await session.commit()
+    await session.refresh(row)
+    return await _suite_out(session, row)
+
+
+@router.get("/api/test-suites/{suite_id}", response_model=TestSuiteOut)
+async def get_test_suite(
+    suite_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestSuite, suite_id)
+    if row is None:
+        raise HTTPException(404, "测试集不存在")
+    return await _suite_out(session, row)
+
+
+@router.put("/api/test-suites/{suite_id}", response_model=TestSuiteOut)
+async def update_test_suite(
+    suite_id: str,
+    body: TestSuiteUpdate,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestSuite, suite_id)
+    if row is None:
+        raise HTTPException(404, "测试集不存在")
+    row.name = body.name.strip()
+    row.description = body.description.strip()
+    await session.commit()
+    await session.refresh(row)
+    return await _suite_out(session, row)
+
+
+@router.delete("/api/test-suites/{suite_id}")
+async def delete_test_suite(
+    suite_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestSuite, suite_id)
+    if row is None:
+        raise HTTPException(404, "测试集不存在")
+    await session.delete(row)
+    await session.commit()
+    return {"ok": True}
+
+
+@router.post("/api/test-suites/{suite_id}/duplicate", response_model=TestSuiteOut)
+async def duplicate_test_suite(
+    suite_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    source = await session.get(TestSuite, suite_id)
+    if source is None:
+        raise HTTPException(404, "测试集不存在")
+    copied = TestSuite(
+        id=_new_id("suite"),
+        name=f"{source.name}（副本）",
+        description=source.description,
+    )
+    session.add(copied)
+    source_cases = (
+        await session.execute(
+            select(TestCase)
+            .where(TestCase.suite_id == suite_id)
+            .order_by(TestCase.sort_order)
+        )
+    ).scalars().all()
+    for index, source_case in enumerate(source_cases):
+        session.add(
+            TestCase(
+                id=_new_id("tc"),
+                suite_id=copied.id,
+                name=source_case.name,
+                description=source_case.description,
+                input_mode=source_case.input_mode,
+                definition=dict(source_case.definition or {}),
+                sort_order=index,
+                last_result="not_run",
+            )
+        )
+    await session.commit()
+    await session.refresh(copied)
+    return await _suite_out(session, copied)
+
+
+@router.get("/api/test-cases", response_model=list[TestCaseOut])
+async def list_test_cases(
+    suite_id: str | None = Query(default=None, alias="suiteId"),
+    session: AsyncSession = Depends(get_session),
+):
+    statement = select(TestCase)
+    if suite_id:
+        statement = statement.where(TestCase.suite_id == suite_id)
+    rows = (
+        await session.execute(
+            statement.order_by(TestCase.suite_id, TestCase.sort_order, TestCase.name)
+        )
+    ).scalars().all()
+    return [_case_out(row) for row in rows]
+
+
+@router.post("/api/test-suites/{suite_id}/cases", response_model=TestCaseOut)
+async def create_test_case(
+    suite_id: str,
+    body: TestCaseWrite,
+    session: AsyncSession = Depends(get_session),
+):
+    if await session.get(TestSuite, suite_id) is None:
+        raise HTTPException(404, "测试集不存在")
+    max_order = await session.scalar(
+        select(func.max(TestCase.sort_order)).where(TestCase.suite_id == suite_id)
+    )
+    row = TestCase(
+        id=_new_id("tc"),
+        suite_id=suite_id,
+        name=body.name.strip(),
+        description=body.description.strip(),
+        input_mode=body.input_mode,
+        definition=body.definition().model_dump(mode="json", by_alias=True),
+        sort_order=int(max_order if max_order is not None else -1) + 1,
+        last_result="not_run",
+    )
+    session.add(row)
+    await _touch_suite(session, suite_id)
+    await session.commit()
+    await session.refresh(row)
+    return _case_out(row)
+
+
+@router.post("/api/test-cases/bulk-delete")
+async def bulk_delete_test_cases(
+    body: TestCaseBulkDeleteIn,
+    session: AsyncSession = Depends(get_session),
+):
+    rows = (
+        await session.execute(select(TestCase).where(TestCase.id.in_(body.case_ids)))
+    ).scalars().all()
+    suite_ids = {row.suite_id for row in rows}
+    for row in rows:
+        await session.delete(row)
+    await session.flush()
+    for suite_id in suite_ids:
+        await _renumber_cases(session, suite_id)
+        await _touch_suite(session, suite_id)
+    await session.commit()
+    return {"ok": True, "deleted": len(rows)}
+
+
+@router.put("/api/test-suites/{suite_id}/case-order")
+async def reorder_test_cases(
+    suite_id: str,
+    body: TestCaseOrderIn,
+    session: AsyncSession = Depends(get_session),
+):
+    rows = (
+        await session.execute(
+            select(TestCase)
+            .where(TestCase.suite_id == suite_id)
+            .order_by(TestCase.sort_order)
+        )
+    ).scalars().all()
+    if {row.id for row in rows} != set(body.case_ids) or len(rows) != len(body.case_ids):
+        raise HTTPException(422, "排序列表必须包含测试集内全部用例且不能重复")
+    by_id = {row.id: row for row in rows}
+    offset = len(rows) + 1
+    for index, row in enumerate(rows):
+        row.sort_order = offset + index
+    await session.flush()
+    for index, case_id in enumerate(body.case_ids):
+        by_id[case_id].sort_order = index
+    await _touch_suite(session, suite_id)
+    await session.commit()
+    return {"ok": True}
+
+
+@router.get("/api/test-cases/{case_id}", response_model=TestCaseOut)
+async def get_test_case(
+    case_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestCase, case_id)
+    if row is None:
+        raise HTTPException(404, "测试用例不存在")
+    return _case_out(row)
+
+
+@router.put("/api/test-cases/{case_id}", response_model=TestCaseOut)
+async def update_test_case(
+    case_id: str,
+    body: TestCaseWrite,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestCase, case_id)
+    if row is None:
+        raise HTTPException(404, "测试用例不存在")
+    _apply_case_write(row, body)
+    await _touch_suite(session, row.suite_id)
+    await session.commit()
+    await session.refresh(row)
+    return _case_out(row)
+
+
+@router.delete("/api/test-cases/{case_id}")
+async def delete_test_case(
+    case_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    row = await session.get(TestCase, case_id)
+    if row is None:
+        raise HTTPException(404, "测试用例不存在")
+    suite_id = row.suite_id
+    await session.delete(row)
+    await session.flush()
+    await _renumber_cases(session, suite_id)
+    await _touch_suite(session, suite_id)
+    await session.commit()
+    return {"ok": True}
+
+
+@router.post("/api/test-cases/{case_id}/duplicate", response_model=TestCaseOut)
+async def duplicate_test_case(
+    case_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    source = await session.get(TestCase, case_id)
+    if source is None:
+        raise HTTPException(404, "测试用例不存在")
+    max_order = await session.scalar(
+        select(func.max(TestCase.sort_order)).where(
+            TestCase.suite_id == source.suite_id
+        )
+    )
+    copied = TestCase(
+        id=_new_id("tc"),
+        suite_id=source.suite_id,
+        name=f"{source.name}（副本）",
+        description=source.description,
+        input_mode=source.input_mode,
+        definition=dict(source.definition or {}),
+        sort_order=int(max_order if max_order is not None else -1) + 1,
+        last_result="not_run",
+    )
+    session.add(copied)
+    await _touch_suite(session, source.suite_id)
+    await session.commit()
+    await session.refresh(copied)
+    return _case_out(copied)
@@ -0,0 +1,240 @@
+"""Start, inspect, and cancel persisted batch text test runs."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from uuid import uuid4
+
+from db.models import (
+    Assistant,
+    ModelResource,
+    TestCase,
+    TestRun,
+    TestRunCase,
+    TestSuite,
+)
+from db.session import get_session
+from fastapi import APIRouter, Depends, HTTPException
+from services.auth import require_admin
+from services.config_resolver import resolve_runtime_config
+from services.test_runs.orchestrator import test_run_orchestrator
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+from test_schemas import (
+    BatchExecutionError,
+    BatchRunCaseOut,
+    BatchRunCreate,
+    BatchRunSnapshotOut,
+    BatchTurnResult,
+    BatchEvaluationResult,
+    TestCaseDefinition,
+)
+
+
+router = APIRouter(
+    prefix="/api/test-runs",
+    tags=["tests"],
+    dependencies=[Depends(require_admin)],
+)
+
+
+def _new_id(prefix: str) -> str:
+    return f"{prefix}_{uuid4().hex}"
+
+
+async def _run_out(
+    session: AsyncSession,
+    run: TestRun,
+) -> BatchRunSnapshotOut:
+    rows = (
+        await session.execute(
+            select(TestRunCase)
+            .where(TestRunCase.run_id == run.id)
+            .order_by(TestRunCase.position)
+        )
+    ).scalars().all()
+    cases: list[BatchRunCaseOut] = []
+    for row in rows:
+        result = dict(row.result or {})
+        cases.append(
+            BatchRunCaseOut(
+                id=str(row.test_case_id or row.case_snapshot.get("id") or row.id),
+                name=row.test_case_name,
+                status=row.status,
+                turns=[
+                    BatchTurnResult.model_validate(item)
+                    for item in result.get("turns") or []
+                ],
+                overall_criteria=[
+                    BatchEvaluationResult.model_validate(item)
+                    for item in result.get("overallCriteria")
+                    or result.get("overall_criteria")
+                    or []
+                ],
+                attempt_count=row.attempt_count,
+                max_attempts=row.max_attempts,
+                execution_error=(
+                    BatchExecutionError.model_validate(row.execution_error)
+                    if row.execution_error
+                    else None
+                ),
+            )
+        )
+    return BatchRunSnapshotOut(
+        id=run.id,
+        status=run.status,
+        title=run.title,
+        assistant_name=run.assistant_name,
+        config=dict(run.config or {}),
+        cases=cases,
+        started_at=run.started_at or run.created_at,
+        finished_at=run.finished_at,
+        stop_reason=run.stop_reason,
+    )
+
+
+@router.post("", response_model=BatchRunSnapshotOut)
+async def create_test_run(
+    body: BatchRunCreate,
+    session: AsyncSession = Depends(get_session),
+):
+    assistant = await session.get(Assistant, body.assistant_id)
+    if assistant is None:
+        raise HTTPException(404, "被测助手不存在")
+    if assistant.runtime_mode != "pipeline":
+        raise HTTPException(422, "第一版批量测试只支持 Pipeline 运行模式")
+    if assistant.type not in {"prompt", "workflow"}:
+        raise HTTPException(422, f"第一版批量测试暂不支持 {assistant.type} 类型助手")
+
+    evaluator_resource = await session.get(
+        ModelResource,
+        body.evaluator_model_resource_id,
+    )
+    if (
+        evaluator_resource is None
+        or evaluator_resource.capability != "LLM"
+        or not evaluator_resource.enabled
+    ):
+        raise HTTPException(422, "评估模型不存在、已停用或不是 LLM 资源")
+    try:
+        runtime_config = await resolve_runtime_config(session, assistant.id)
+    except ValueError as exc:
+        raise HTTPException(422, str(exc)) from exc
+
+    rows = (
+        await session.execute(select(TestCase).where(TestCase.id.in_(body.case_ids)))
+    ).scalars().all()
+    by_id = {row.id: row for row in rows}
+    missing = [case_id for case_id in body.case_ids if case_id not in by_id]
+    if missing:
+        raise HTTPException(422, "测试用例不存在: " + "、".join(missing))
+    ordered_cases = [by_id[case_id] for case_id in body.case_ids]
+    tools_by_id = {tool.id: tool for tool in runtime_config.tools}
+    for row in ordered_cases:
+        if row.input_mode != "fixed_script_text":
+            raise HTTPException(422, f"用例“{row.name}”不是可运行的固定文字模式")
+        try:
+            definition = TestCaseDefinition.model_validate(row.definition or {})
+        except ValueError as exc:
+            raise HTTPException(422, f"用例“{row.name}”校验失败: {exc}") from exc
+        for turn in definition.turns:
+            for behavior in turn.behaviors:
+                if behavior.type != "tool_call":
+                    continue
+                tool = tools_by_id.get(behavior.tool_id)
+                if tool is None or tool.function_name != behavior.function_name:
+                    raise HTTPException(
+                        422,
+                        f"用例“{row.name}”引用的工具 {behavior.function_name} "
+                        "不属于当前被测助手",
+                    )
+
+    suite_ids = list(dict.fromkeys(row.suite_id for row in ordered_cases))
+    suites = (
+        await session.execute(select(TestSuite).where(TestSuite.id.in_(suite_ids)))
+    ).scalars().all()
+    suite_by_id = {row.id: row for row in suites}
+    if body.title:
+        title = body.title.strip()
+    elif len(suite_ids) == 1 and suite_ids[0] in suite_by_id:
+        title = f"{suite_by_id[suite_ids[0]].name} · {len(ordered_cases)} 个用例"
+    else:
+        title = f"批量测试 · {len(ordered_cases)} 个用例"
+
+    now = datetime.now(UTC)
+    run = TestRun(
+        id=_new_id("run"),
+        assistant_id=assistant.id,
+        assistant_name=assistant.name,
+        title=title,
+        status="queued",
+        config={
+            "suiteCount": len(suite_ids),
+            "evaluatorModelResourceId": evaluator_resource.id,
+            "evaluatorModelResourceName": evaluator_resource.name,
+            "evaluatorModel": str(
+                (evaluator_resource.values or {}).get("modelId") or ""
+            ),
+            **body.config.model_dump(mode="json", by_alias=True),
+        },
+        started_at=now,
+    )
+    session.add(run)
+    for position, row in enumerate(ordered_cases):
+        definition = TestCaseDefinition.model_validate(row.definition or {})
+        snapshot = {
+            "id": row.id,
+            "suiteId": row.suite_id,
+            "name": row.name,
+            "description": row.description,
+            "inputMode": row.input_mode,
+            **definition.model_dump(mode="json", by_alias=True),
+        }
+        session.add(
+            TestRunCase(
+                id=_new_id("rc"),
+                run_id=run.id,
+                test_case_id=row.id,
+                test_case_name=row.name,
+                suite_id=row.suite_id,
+                position=position,
+                case_snapshot=snapshot,
+                status="waiting",
+                attempt_count=0,
+                max_attempts=body.config.error_retry_count + 1,
+                result={"turns": [], "overallCriteria": []},
+            )
+        )
+    await session.commit()
+    await session.refresh(run)
+    test_run_orchestrator.start(run.id)
+    return await _run_out(session, run)
+
+
+@router.get("/{run_id}", response_model=BatchRunSnapshotOut)
+async def get_test_run(
+    run_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    run = await session.get(TestRun, run_id)
+    if run is None:
+        raise HTTPException(404, "批量测试运行不存在")
+    return await _run_out(session, run)
+
+
+@router.post("/{run_id}/cancel", response_model=BatchRunSnapshotOut)
+async def cancel_test_run(
+    run_id: str,
+    session: AsyncSession = Depends(get_session),
+):
+    run = await session.get(TestRun, run_id)
+    if run is None:
+        raise HTTPException(404, "批量测试运行不存在")
+    if run.status in {"completed", "cancelled"}:
+        return await _run_out(session, run)
+    run.cancel_requested = True
+    run.stop_reason = "manual"
+    await session.commit()
+    await test_run_orchestrator.cancel(run_id)
+    await session.refresh(run)
+    return await _run_out(session, run)