wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(testing): add fixed tool simulations

Browse Source
This commit is contained in:
Xin Wang
2026-08-26 09:36:37 +08:00
parent 26582bfb87
commit 239b9027ca
16 changed files with 1525 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/data/batch-run.ts
+10
View File
@@ -31,6 +31,16 @@ export type BatchExecutionError = {
retryable: boolean;
};
const TEST_EXECUTION_ERROR_LABEL: Record<string, string> = {
UNMOCKED_TOOL_CALL: "工具未配置模拟",
TOOL_SIMULATION_EXHAUSTED: "工具模拟响应已耗尽",
TOOL_SIMULATION_SCOPE_MISMATCH: "工具模拟阶段不匹配",
};
export function testExecutionErrorLabel(code: string): string {
return TEST_EXECUTION_ERROR_LABEL[code] ?? code;
}
export type BatchEvaluationResult = {
id: string;
label: string;
frontend/src/data/test-suites.ts
+185 -31
View File
@@ -56,13 +56,37 @@ export type ReplyExpectedBehavior = {
llmCriteria: string;
};
type ToolMockResponse = {
export type ToolSimulationResponse = {
outcome: "success" | "error";
/** JSON 文本，运行时会作为工具返回值注入 pipeline */
body: string;
delayMs: number;
};
export type ToolSimulationScope =
| { phase: "any" }
| { phase: "startup" }
| { phase: "turn"; turnId: string };
export type ToolSimulationStep = ToolSimulationResponse & {
id: string;
scope: ToolSimulationScope;
};
export type ToolSimulation = {
toolId: string;
functionName: string;
} & (
| {
mode: "fixed";
response: ToolSimulationResponse;
}
| {
mode: "sequence";
steps: ToolSimulationStep[];
}
);
/** 预期行为：工具调用或禁止调用指定工具 */
export type ToolCallExpectedBehavior = {
id: string;
@@ -74,8 +98,6 @@ export type ToolCallExpectedBehavior = {
maxCalls: number | null;
/** 仅包含用户主动配置的参数；未列出的参数不校验 */
paramAssertions: ToolParamAssertion[];
/** 调用工具后注入 pipeline 的稳定返回值 */
mockResponse: ToolMockResponse;
};
export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
@@ -132,6 +154,8 @@ export type TestCase = {
contextTurns: ContextTurn[];
/** 固定脚本轮次 */
turns: FixedInputTurn[];
/** 测试用例级工具模拟；与工具调用断言相互独立 */
toolSimulations: ToolSimulation[];
/** 整段对话业务目标预期；所有标准均通过才算用例通过 */
overallCriteria: OverallCriterion[];
/** 套件内排序，越小越靠前 */
@@ -148,7 +172,11 @@ export const DEFAULT_INPUT_MODE: TestCaseInputMode = "fixed_script_text";
export function getTestCaseValidationMessage(
item: Pick<
TestCase,
"inputMode" | "contextTurns" | "turns" | "overallCriteria"
| "inputMode"
| "contextTurns"
| "turns"
| "toolSimulations"
| "overallCriteria"
>,
): string | null {
if (item.inputMode !== "fixed_script_text") {
@@ -248,6 +276,49 @@ export function getTestCaseValidationMessage(
}
}
}
const simulationToolIds = new Set<string>();
const simulationStepIds = new Set<string>();
const turnIds = new Set(item.turns.map((turn) => turn.id));
for (let index = 0; index < item.toolSimulations.length; index += 1) {
const simulation = item.toolSimulations[index];
if (simulationToolIds.has(simulation.toolId)) {
return `工具模拟 ${index + 1}：同一工具只能配置一条模拟`;
}
simulationToolIds.add(simulation.toolId);
const message = getToolSimulationValidationMessage(
simulation,
turnIds,
simulationStepIds,
);
if (message) return `工具模拟 ${index + 1}：${message}`;
}
for (let turnIndex = 0; turnIndex < item.turns.length; turnIndex += 1) {
const turn = item.turns[turnIndex];
for (const behavior of turn.behaviors) {
if (behavior.type !== "tool_call" || behavior.expectation !== "called") {
continue;
}
const simulation = item.toolSimulations.find(
(candidate) => candidate.toolId === behavior.toolId,
);
if (!simulation) {
return `第 ${turnIndex + 1} 轮工具 ${behavior.functionName} 缺少工具模拟`;
}
if (simulation.functionName !== behavior.functionName) {
return `第 ${turnIndex + 1} 轮工具 ${behavior.functionName} 的断言与模拟不匹配`;
}
if (
simulation.mode === "sequence" &&
!simulation.steps.some(
(step) =>
step.scope.phase === "any" ||
(step.scope.phase === "turn" && step.scope.turnId === turn.id),
)
) {
return `第 ${turnIndex + 1} 轮工具 ${behavior.functionName} 缺少覆盖本轮的模拟步骤`;
}
}
}
for (let index = 0; index < item.overallCriteria.length; index += 1) {
const criterion = item.overallCriteria[index];
if (!criterion.name.trim()) {
@@ -269,7 +340,11 @@ export function getTestCaseValidationMessage(
export function isTestCaseRunnable(
item: Pick<
TestCase,
"inputMode" | "contextTurns" | "turns" | "overallCriteria"
| "inputMode"
| "contextTurns"
| "turns"
| "toolSimulations"
| "overallCriteria"
>,
): boolean {
return getTestCaseValidationMessage(item) === null;
@@ -312,7 +387,55 @@ export const TOOL_PARAM_MATCH_MODE_LABEL: Record<ToolParamMatchMode, string> = {
};
export const OVERALL_CRITERION_TEXT_MAX = 2000;
const TOOL_MOCK_DELAY_MAX_MS = 60_000;
const TOOL_SIMULATION_DELAY_MAX_MS = 60_000;
function getSimulationResponseValidationMessage(
response: ToolSimulationResponse,
): string | null {
if (!response.body.trim()) return "请填写工具返回值";
try {
JSON.parse(response.body);
} catch {
return "工具返回值必须是有效 JSON";
}
if (
!Number.isInteger(response.delayMs) ||
response.delayMs < 0 ||
response.delayMs > TOOL_SIMULATION_DELAY_MAX_MS
) {
return "模拟延迟需为 0–60000 毫秒的整数";
}
return null;
}
export function getToolSimulationValidationMessage(
simulation: ToolSimulation,
turnIds: Set<string>,
seenStepIds: Set<string> = new Set(),
): string | null {
if (!simulation.toolId.trim() || !simulation.functionName.trim()) {
return "请选择有效工具";
}
if (simulation.mode === "fixed") {
return getSimulationResponseValidationMessage(simulation.response);
}
if (simulation.steps.length === 0) return "顺序返回至少需要一个步骤";
for (let index = 0; index < simulation.steps.length; index += 1) {
const step = simulation.steps[index];
if (!step.id.trim()) return `步骤 ${index + 1} 缺少 ID`;
if (seenStepIds.has(step.id)) return `步骤 ID 重复：${step.id}`;
seenStepIds.add(step.id);
if (
step.scope.phase === "turn" &&
!turnIds.has(step.scope.turnId)
) {
return `步骤 ${index + 1} 引用了不存在的轮次`;
}
const message = getSimulationResponseValidationMessage(step);
if (message) return `步骤 ${index + 1}：${message}`;
}
return null;
}
export function getExpectedBehaviorValidationMessage(
behavior: ExpectedBehavior,
@@ -353,21 +476,6 @@ export function getExpectedBehaviorValidationMessage(
(item) => !item.name.trim() || !item.value.trim(),
);
if (emptyParam) return "工具参数断言的名称和值不能为空";
if (!behavior.mockResponse.body.trim()) {
return "请填写 Mock 工具返回值";
}
try {
JSON.parse(behavior.mockResponse.body);
} catch {
return "Mock 工具返回值必须是有效 JSON";
}
if (
!Number.isInteger(behavior.mockResponse.delayMs) ||
behavior.mockResponse.delayMs < 0 ||
behavior.mockResponse.delayMs > TOOL_MOCK_DELAY_MAX_MS
) {
return "Mock 延迟需为 0–60000 毫秒的整数";
}
}
return null;
}
@@ -387,6 +495,29 @@ function nextCriterionId() {
return newEditorId("criterion");
}
export function createToolSimulation(input: {
toolId: string;
functionName: string;
toolType: "http" | "mcp" | "client";
}): ToolSimulation {
const body =
input.toolType === "client"
? input.functionName === "show_message"
? '{\n "action": "confirmed"\n}'
: '{\n "action": "selected",\n "value": ""\n}'
: '{\n "status": "ok"\n}';
return {
toolId: input.toolId,
functionName: input.functionName,
mode: "fixed",
response: {
outcome: "success",
body,
delayMs: 0,
},
};
}
export function createOverallCriterion(
name = "新的评估标准",
): OverallCriterion {
@@ -453,11 +584,6 @@ export function createToolCallBehavior(input?: {
minCalls: 1,
maxCalls: null,
paramAssertions: [],
mockResponse: {
outcome: "success",
body: '{\n "status": "ok"\n}',
delayMs: 0,
},
};
}
@@ -482,11 +608,6 @@ function cloneBehavior(behavior: ExpectedBehavior): ExpectedBehavior {
minCalls: behavior.minCalls,
maxCalls: behavior.maxCalls,
paramAssertions: behavior.paramAssertions.map((item) => ({ ...item })),
mockResponse: {
outcome: behavior.mockResponse.outcome,
body: behavior.mockResponse.body,
delayMs: behavior.mockResponse.delayMs,
},
};
}
@@ -499,6 +620,39 @@ export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
}));
}
export function prepareFixedToolSimulations(
values: ToolSimulation[] | null | undefined,
): ToolSimulation[] {
return (values ?? []).map((simulation) =>
simulation.mode === "fixed"
? {
toolId: simulation.toolId,
functionName: simulation.functionName,
mode: "fixed",
response: { ...simulation.response },
}
: {
toolId: simulation.toolId,
functionName: simulation.functionName,
mode: "fixed",
response: simulation.steps[0]
? {
outcome: simulation.steps[0].outcome,
body: simulation.steps[0].body,
delayMs: simulation.steps[0].delayMs,
}
: {
outcome: "success",
body:
simulation.functionName === "show_message"
? '{\n "action": "confirmed"\n}'
: '{\n "status": "ok"\n}',
delayMs: 0,
},
},
);
}
export function formatUpdatedAt(value?: string | null) {
if (!value) return "—";
const date = new Date(value);