Xin Wang
7560f9a525
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types. - Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages. - Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs. - Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases. - Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
399 lines
14 KiB
Python
399 lines
14 KiB
Python
"""Contracts for persisted text test cases and batch execution results."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from datetime import datetime
|
||
from typing import Annotated, Any, Literal, Union
|
||
|
||
from pydantic import Field, field_validator, model_validator
|
||
|
||
from schemas import CamelModel
|
||
|
||
|
||
TestCaseInputMode = Literal[
|
||
"fixed_script_text",
|
||
"fixed_script_turn_voice",
|
||
"fixed_script_continuous_voice",
|
||
"user_sim_text",
|
||
"user_sim_voice",
|
||
]
|
||
ContextRole = Literal["agent", "user", "tool_call", "tool_result"]
|
||
EvaluationKind = Literal["reply", "tool_call", "overall"]
|
||
EvaluationStatus = Literal["pass", "fail"]
|
||
CaseStatus = Literal["waiting", "running", "pass", "fail", "error", "skipped"]
|
||
RunStatus = Literal["queued", "running", "completed", "cancelled"]
|
||
ErrorStage = Literal["pipeline", "model", "tool", "evaluation"]
|
||
StopReason = Literal["manual", "assertion_failure", "execution_error"]
|
||
|
||
|
||
class ContextTurn(CamelModel):
|
||
role: ContextRole
|
||
content: str = Field(max_length=20_000)
|
||
tool_name: str | None = Field(default=None, max_length=128)
|
||
tool_call_id: str | None = Field(default=None, max_length=128)
|
||
is_error: bool | None = None
|
||
|
||
@model_validator(mode="after")
|
||
def validate_content(self):
|
||
self.content = self.content.strip()
|
||
self.tool_call_id = (self.tool_call_id or "").strip() or None
|
||
if not self.content:
|
||
raise ValueError("上下文内容不能为空")
|
||
if self.role in {"tool_call", "tool_result"}:
|
||
self.tool_name = (self.tool_name or "").strip()
|
||
if not self.tool_name:
|
||
raise ValueError("工具上下文必须填写工具名称")
|
||
try:
|
||
payload = json.loads(self.content)
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError("工具上下文必须是有效 JSON") from exc
|
||
if self.role == "tool_call" and not isinstance(payload, dict):
|
||
raise ValueError("Tool Call 参数必须是 JSON 对象")
|
||
return self
|
||
|
||
|
||
class ToolParamAssertion(CamelModel):
|
||
name: str = Field(min_length=1, max_length=128)
|
||
match_mode: Literal["exact", "regex", "llm"]
|
||
value: str = Field(min_length=1, max_length=4_000)
|
||
|
||
@field_validator("name", "value")
|
||
@classmethod
|
||
def strip_required_text(cls, value: str) -> str:
|
||
value = value.strip()
|
||
if not value:
|
||
raise ValueError("参数名称和值不能为空")
|
||
return value
|
||
|
||
@model_validator(mode="after")
|
||
def validate_regex(self):
|
||
if self.match_mode == "regex":
|
||
try:
|
||
re.compile(self.value)
|
||
except re.error as exc:
|
||
raise ValueError(f"正则表达式无效: {exc}") from exc
|
||
return self
|
||
|
||
|
||
class ToolMockResponse(CamelModel):
|
||
outcome: Literal["success", "error"] = "success"
|
||
body: str = Field(default='{"status":"ok"}', max_length=100_000)
|
||
delay_ms: int = Field(default=0, ge=0, le=60_000)
|
||
|
||
@model_validator(mode="after")
|
||
def validate_json_body(self):
|
||
if not self.body.strip():
|
||
raise ValueError("Mock 工具返回值不能为空")
|
||
try:
|
||
json.loads(self.body)
|
||
except json.JSONDecodeError as exc:
|
||
raise ValueError("Mock 工具返回值必须是有效 JSON") from exc
|
||
return self
|
||
|
||
|
||
class ReplyExpectedBehavior(CamelModel):
|
||
id: str = Field(min_length=1, max_length=128)
|
||
type: Literal["reply"] = "reply"
|
||
assertion_type: Literal["keyword", "llm"]
|
||
keywords: list[str] = Field(default_factory=list, max_length=50)
|
||
keyword_match_mode: Literal["any", "all"] = "any"
|
||
negate_keywords: bool = False
|
||
llm_criteria: str = Field(default="", max_length=4_000)
|
||
|
||
@model_validator(mode="after")
|
||
def validate_assertion(self):
|
||
self.keywords = [item.strip() for item in self.keywords if item.strip()]
|
||
self.llm_criteria = self.llm_criteria.strip()
|
||
if self.assertion_type == "keyword" and not self.keywords:
|
||
raise ValueError("关键词断言至少需要一个关键词")
|
||
if self.assertion_type == "llm" and not self.llm_criteria:
|
||
raise ValueError("LLM 判断要求不能为空")
|
||
return self
|
||
|
||
|
||
class ToolCallExpectedBehavior(CamelModel):
|
||
id: str = Field(min_length=1, max_length=128)
|
||
type: Literal["tool_call"] = "tool_call"
|
||
tool_id: str = Field(min_length=1, max_length=40)
|
||
function_name: str = Field(min_length=1, max_length=128)
|
||
expectation: Literal["called", "not_called"]
|
||
min_calls: int = Field(default=1, ge=0, le=100)
|
||
max_calls: int | None = Field(default=None, ge=0, le=100)
|
||
param_assertions: list[ToolParamAssertion] = Field(default_factory=list, max_length=50)
|
||
mock_response: ToolMockResponse = Field(default_factory=ToolMockResponse)
|
||
|
||
@model_validator(mode="after")
|
||
def validate_call_expectation(self):
|
||
self.tool_id = self.tool_id.strip()
|
||
self.function_name = self.function_name.strip()
|
||
if not self.tool_id or not self.function_name:
|
||
raise ValueError("必须选择有效的工具")
|
||
if self.expectation == "called" and self.min_calls < 1:
|
||
raise ValueError("应调用工具时最少调用次数必须大于 0")
|
||
if self.max_calls is not None and self.max_calls < self.min_calls:
|
||
raise ValueError("最多调用次数不能小于最少调用次数")
|
||
return self
|
||
|
||
|
||
ExpectedBehavior = Annotated[
|
||
Union[ReplyExpectedBehavior, ToolCallExpectedBehavior],
|
||
Field(discriminator="type"),
|
||
]
|
||
|
||
|
||
class FixedInputTurn(CamelModel):
|
||
id: str = Field(min_length=1, max_length=128)
|
||
user_input: str = Field(min_length=1, max_length=20_000)
|
||
behaviors: list[ExpectedBehavior] = Field(default_factory=list, max_length=100)
|
||
|
||
@field_validator("user_input")
|
||
@classmethod
|
||
def strip_user_input(cls, value: str) -> str:
|
||
value = value.strip()
|
||
if not value:
|
||
raise ValueError("用户输入不能为空")
|
||
return value
|
||
|
||
|
||
class OverallCriterion(CamelModel):
|
||
id: str = Field(min_length=1, max_length=128)
|
||
type: Literal["llm"] = "llm"
|
||
name: str = Field(min_length=1, max_length=80)
|
||
criteria: str = Field(min_length=1, max_length=2_000)
|
||
|
||
@field_validator("name", "criteria")
|
||
@classmethod
|
||
def strip_criterion(cls, value: str) -> str:
|
||
value = value.strip()
|
||
if not value:
|
||
raise ValueError("整体评估标准不能为空")
|
||
return value
|
||
|
||
|
||
class TestCaseDefinition(CamelModel):
|
||
context_turns: list[ContextTurn] = Field(default_factory=list, max_length=100)
|
||
turns: list[FixedInputTurn] = Field(min_length=1, max_length=100)
|
||
overall_criteria: list[OverallCriterion] = Field(default_factory=list, max_length=50)
|
||
|
||
@model_validator(mode="after")
|
||
def validate_has_expectation(self):
|
||
if not any(turn.behaviors for turn in self.turns) and not self.overall_criteria:
|
||
raise ValueError("至少配置一项有效的预期行为或整体评估标准")
|
||
return self
|
||
|
||
@model_validator(mode="after")
|
||
def validate_tool_context_pairs(self):
|
||
pending: list[tuple[str | None, str]] = []
|
||
seen_ids: set[str] = set()
|
||
for index, turn in enumerate(self.context_turns):
|
||
if turn.role == "tool_call":
|
||
call_id = (turn.tool_call_id or "").strip() or None
|
||
if call_id and call_id in seen_ids:
|
||
raise ValueError(f"上下文第 {index + 1} 条 Tool Call ID 重复")
|
||
if call_id:
|
||
seen_ids.add(call_id)
|
||
pending.append((call_id, turn.tool_name or ""))
|
||
continue
|
||
if turn.role != "tool_result":
|
||
continue
|
||
|
||
result_id = (turn.tool_call_id or "").strip() or None
|
||
matched_index = -1
|
||
for pending_index in range(len(pending) - 1, -1, -1):
|
||
call_id, tool_name = pending[pending_index]
|
||
if result_id:
|
||
matches = call_id == result_id
|
||
else:
|
||
matches = tool_name == turn.tool_name
|
||
if matches:
|
||
matched_index = pending_index
|
||
break
|
||
if matched_index < 0:
|
||
raise ValueError(
|
||
f"上下文第 {index + 1} 条 Tool Result 没有匹配的 Tool Call"
|
||
)
|
||
_call_id, tool_name = pending.pop(matched_index)
|
||
if tool_name != turn.tool_name:
|
||
raise ValueError(
|
||
f"上下文第 {index + 1} 条 Tool Result 的工具名称不匹配"
|
||
)
|
||
|
||
if pending:
|
||
names = "、".join(tool_name for _call_id, tool_name in pending)
|
||
raise ValueError(f"上下文 Tool Call 缺少对应的 Tool Result：{names}")
|
||
return self
|
||
|
||
|
||
class TestSuiteCreate(CamelModel):
|
||
name: str = Field(min_length=1, max_length=128)
|
||
description: str = Field(default="", max_length=2_048)
|
||
|
||
@field_validator("name")
|
||
@classmethod
|
||
def strip_name(cls, value: str) -> str:
|
||
value = value.strip()
|
||
if not value:
|
||
raise ValueError("测试集名称不能为空")
|
||
return value
|
||
|
||
|
||
class TestSuiteUpdate(TestSuiteCreate):
|
||
pass
|
||
|
||
|
||
class TestSuiteOut(CamelModel):
|
||
id: str
|
||
name: str
|
||
description: str
|
||
case_count: int = 0
|
||
passed_count: int = 0
|
||
run_count: int = 0
|
||
updated_at: datetime
|
||
|
||
|
||
class TestCaseWrite(CamelModel):
|
||
name: str = Field(min_length=1, max_length=128)
|
||
description: str = Field(default="", max_length=2_048)
|
||
input_mode: TestCaseInputMode = "fixed_script_text"
|
||
context_turns: list[ContextTurn] = Field(default_factory=list, max_length=100)
|
||
turns: list[FixedInputTurn] = Field(min_length=1, max_length=100)
|
||
overall_criteria: list[OverallCriterion] = Field(default_factory=list, max_length=50)
|
||
|
||
@field_validator("name")
|
||
@classmethod
|
||
def strip_name(cls, value: str) -> str:
|
||
value = value.strip()
|
||
if not value:
|
||
raise ValueError("测试用例名称不能为空")
|
||
return value
|
||
|
||
@model_validator(mode="after")
|
||
def validate_mvp_definition(self):
|
||
if self.input_mode != "fixed_script_text":
|
||
raise ValueError("第一版只支持固定脚本 · 文字")
|
||
TestCaseDefinition(
|
||
context_turns=self.context_turns,
|
||
turns=self.turns,
|
||
overall_criteria=self.overall_criteria,
|
||
)
|
||
return self
|
||
|
||
def definition(self) -> TestCaseDefinition:
|
||
return TestCaseDefinition(
|
||
context_turns=self.context_turns,
|
||
turns=self.turns,
|
||
overall_criteria=self.overall_criteria,
|
||
)
|
||
|
||
|
||
class TestCaseOut(TestCaseWrite):
|
||
id: str
|
||
suite_id: str
|
||
last_result: Literal["pass", "fail", "not_run"]
|
||
sort_order: int
|
||
updated_at: datetime
|
||
|
||
|
||
class TestCaseOrderIn(CamelModel):
|
||
case_ids: list[str] = Field(min_length=1, max_length=1_000)
|
||
|
||
|
||
class TestCaseBulkDeleteIn(CamelModel):
|
||
case_ids: list[str] = Field(min_length=1, max_length=1_000)
|
||
|
||
|
||
class BatchRunConfig(CamelModel):
|
||
concurrency: int = Field(default=3, ge=1, le=20)
|
||
timeout_secs: int = Field(default=60, ge=1, le=600)
|
||
failure_strategy: Literal["continue", "stop_on_fail"] = "continue"
|
||
error_retry_count: int = Field(default=0, ge=0, le=3)
|
||
error_strategy: Literal["continue", "stop_on_error"] = "continue"
|
||
|
||
|
||
class BatchRunCreate(CamelModel):
|
||
assistant_id: str = Field(min_length=1, max_length=40)
|
||
evaluator_model_resource_id: str = Field(min_length=1, max_length=40)
|
||
case_ids: list[str] = Field(min_length=1, max_length=1_000)
|
||
dynamic_variables: dict[str, str | int | float | bool] = Field(
|
||
default_factory=dict,
|
||
max_length=50,
|
||
)
|
||
config: BatchRunConfig = Field(default_factory=BatchRunConfig)
|
||
title: str | None = Field(default=None, max_length=256)
|
||
|
||
@field_validator("case_ids")
|
||
@classmethod
|
||
def unique_case_ids(cls, values: list[str]) -> list[str]:
|
||
normalized = list(dict.fromkeys(value.strip() for value in values if value.strip()))
|
||
if not normalized:
|
||
raise ValueError("至少选择一个测试用例")
|
||
return normalized
|
||
|
||
|
||
class BatchExecutionError(CamelModel):
|
||
code: str
|
||
message: str
|
||
stage: ErrorStage
|
||
retryable: bool
|
||
|
||
|
||
class BatchEvaluationResult(CamelModel):
|
||
id: str
|
||
label: str
|
||
kind: EvaluationKind
|
||
status: EvaluationStatus
|
||
expected: str
|
||
actual: str
|
||
reason: str = ""
|
||
|
||
|
||
class BatchToolCallRecord(CamelModel):
|
||
id: str
|
||
function_name: str
|
||
arguments_json: str
|
||
result_json: str
|
||
outcome: Literal["success", "error"]
|
||
duration_ms: int = 0
|
||
|
||
|
||
class BatchTurnResult(CamelModel):
|
||
id: str
|
||
index: int
|
||
user_input: str
|
||
assistant_reply: str
|
||
tool_calls: list[BatchToolCallRecord] = Field(default_factory=list)
|
||
evaluations: list[BatchEvaluationResult] = Field(default_factory=list)
|
||
|
||
|
||
class BatchStartupResult(CamelModel):
|
||
completed: bool = False
|
||
greeting: str = ""
|
||
assistant_outputs: list[str] = Field(default_factory=list)
|
||
auto_confirmed_messages: list[dict[str, Any]] = Field(default_factory=list)
|
||
|
||
|
||
class BatchRunCaseOut(CamelModel):
|
||
id: str
|
||
name: str
|
||
status: CaseStatus
|
||
turns: list[BatchTurnResult] = Field(default_factory=list)
|
||
startup: BatchStartupResult = Field(default_factory=BatchStartupResult)
|
||
overall_criteria: list[BatchEvaluationResult] = Field(default_factory=list)
|
||
attempt_count: int
|
||
max_attempts: int
|
||
execution_error: BatchExecutionError | None = None
|
||
|
||
|
||
class BatchRunSnapshotOut(CamelModel):
|
||
id: str
|
||
status: RunStatus
|
||
title: str
|
||
assistant_name: str
|
||
config: dict[str, Any]
|
||
cases: list[BatchRunCaseOut]
|
||
started_at: datetime
|
||
finished_at: datetime | None
|
||
stop_reason: StopReason | None
|