feat: add persisted batch text testing
This commit is contained in:
Xin Wang
+21
-267
@@ -1,14 +1,4 @@
|
||||
/**
|
||||
* 批量测试运行 — 前端 mock。
|
||||
* 真实执行引擎接入前，用本地状态模拟进度与结果。
|
||||
*/
|
||||
|
||||
import type {
|
||||
ExpectedBehavior,
|
||||
ReplyExpectedBehavior,
|
||||
TestCase,
|
||||
ToolCallExpectedBehavior,
|
||||
} from "@/data/test-suites";
|
||||
/** 批量测试运行的前后端共享结果契约与纯展示辅助函数。 */
|
||||
|
||||
export type BatchCaseStatus =
|
||||
| "waiting"
|
||||
@@ -34,24 +24,13 @@ export const BATCH_ERROR_STRATEGY_LABEL: Record<BatchErrorStrategy, string> = {
|
||||
stop_on_error: "标记错误并停止",
|
||||
};
|
||||
|
||||
type BatchExecutionError = {
|
||||
export type BatchExecutionError = {
|
||||
code: string;
|
||||
message: string;
|
||||
stage: "pipeline" | "model" | "tool" | "evaluation";
|
||||
retryable: boolean;
|
||||
};
|
||||
|
||||
export type BatchRunCase = {
|
||||
id: string;
|
||||
name: string;
|
||||
status: BatchCaseStatus;
|
||||
turns: BatchTurnResult[];
|
||||
overallCriteria: BatchEvaluationResult[];
|
||||
attemptCount: number;
|
||||
maxAttempts: number;
|
||||
executionError: BatchExecutionError | null;
|
||||
};
|
||||
|
||||
export type BatchEvaluationResult = {
|
||||
id: string;
|
||||
label: string;
|
||||
@@ -80,13 +59,30 @@ export type BatchTurnResult = {
|
||||
evaluations: BatchEvaluationResult[];
|
||||
};
|
||||
|
||||
export type BatchRunCase = {
|
||||
id: string;
|
||||
name: string;
|
||||
status: BatchCaseStatus;
|
||||
turns: BatchTurnResult[];
|
||||
overallCriteria: BatchEvaluationResult[];
|
||||
attemptCount: number;
|
||||
maxAttempts: number;
|
||||
executionError: BatchExecutionError | null;
|
||||
};
|
||||
|
||||
export type BatchRunPhase = "config" | "running" | "completed";
|
||||
export type BatchRunStatus = "queued" | "running" | "completed" | "cancelled";
|
||||
|
||||
export type BatchRunSnapshot = {
|
||||
id: string;
|
||||
status: BatchRunStatus;
|
||||
title: string;
|
||||
assistantName: string;
|
||||
config: {
|
||||
suiteCount: number;
|
||||
evaluatorModelResourceId: string;
|
||||
evaluatorModelResourceName: string;
|
||||
evaluatorModel: string;
|
||||
concurrency: number;
|
||||
timeoutSecs: number;
|
||||
failureStrategy: BatchFailureStrategy;
|
||||
@@ -99,246 +95,6 @@ export type BatchRunSnapshot = {
|
||||
stopReason: "manual" | "assertion_failure" | "execution_error" | null;
|
||||
};
|
||||
|
||||
export type BatchMockExecutionPlan = Record<string, BatchExecutionError>;
|
||||
|
||||
/** 预定部分用例失败，方便演示失败展开态 */
|
||||
function shouldFail(index: number, item: TestCase): boolean {
|
||||
if (item.lastResult === "fail") return true;
|
||||
return index % 4 === 2;
|
||||
}
|
||||
|
||||
/** 稳定产生少量执行错误，便于检查重试和继续/停止流程。 */
|
||||
function shouldError(index: number): boolean {
|
||||
return index % 5 === 3;
|
||||
}
|
||||
|
||||
function replyExpectation(behavior: ReplyExpectedBehavior): string {
|
||||
if (behavior.assertionType === "llm") return behavior.llmCriteria;
|
||||
const relation = behavior.negateKeywords
|
||||
? "不应包含"
|
||||
: behavior.keywordMatchMode === "all"
|
||||
? "应包含全部"
|
||||
: "应至少包含其一";
|
||||
return `${relation}：${behavior.keywords.join("、")}`;
|
||||
}
|
||||
|
||||
function toolExpectation(behavior: ToolCallExpectedBehavior): string {
|
||||
if (behavior.expectation === "not_called") {
|
||||
return `不应调用 ${behavior.functionName}`;
|
||||
}
|
||||
const count =
|
||||
behavior.maxCalls === null
|
||||
? `${behavior.minCalls} 次以上`
|
||||
: behavior.minCalls === behavior.maxCalls
|
||||
? `${behavior.minCalls} 次`
|
||||
: `${behavior.minCalls}–${behavior.maxCalls} 次`;
|
||||
return `应调用 ${behavior.functionName} ${count}`;
|
||||
}
|
||||
|
||||
function mockAssistantReply(behaviors: ExpectedBehavior[]): string {
|
||||
const positiveKeyword = behaviors.find(
|
||||
(behavior): behavior is ReplyExpectedBehavior =>
|
||||
behavior.type === "reply" &&
|
||||
behavior.assertionType === "keyword" &&
|
||||
!behavior.negateKeywords &&
|
||||
behavior.keywords.length > 0,
|
||||
);
|
||||
if (positiveKeyword) {
|
||||
return `好的，我已了解。请继续说明${positiveKeyword.keywords.slice(0, 2).join("和")}。`;
|
||||
}
|
||||
return "好的，我已记录当前信息，我们继续处理。";
|
||||
}
|
||||
|
||||
function mockToolArguments(behavior: ToolCallExpectedBehavior): string {
|
||||
const entries = behavior.paramAssertions.map((item) => [
|
||||
item.name,
|
||||
item.matchMode === "exact" ? item.value : `mock_${item.name}`,
|
||||
]);
|
||||
return JSON.stringify(Object.fromEntries(entries), null, 2);
|
||||
}
|
||||
|
||||
function createBehaviorEvaluation(
|
||||
behavior: ExpectedBehavior,
|
||||
fail: boolean,
|
||||
): {
|
||||
evaluation: BatchEvaluationResult;
|
||||
toolCalls: BatchToolCallRecord[];
|
||||
} {
|
||||
if (behavior.type === "reply") {
|
||||
const reason = fail
|
||||
? behavior.assertionType === "keyword"
|
||||
? "实际回复未满足关键词规则。"
|
||||
: "LLM 判断认为回复未达到该项语义要求。"
|
||||
: "";
|
||||
return {
|
||||
evaluation: {
|
||||
id: behavior.id,
|
||||
label:
|
||||
behavior.assertionType === "keyword"
|
||||
? "回复 · 关键词"
|
||||
: "回复 · LLM 判断",
|
||||
kind: "reply",
|
||||
status: fail ? "fail" : "pass",
|
||||
expected: replyExpectation(behavior),
|
||||
actual: fail
|
||||
? "已为您转接人工处理。"
|
||||
: "实际回复满足本条内容要求。",
|
||||
reason,
|
||||
},
|
||||
toolCalls: [],
|
||||
};
|
||||
}
|
||||
|
||||
const shouldRecordCall =
|
||||
behavior.expectation === "called" ? !fail : fail;
|
||||
const toolCalls: BatchToolCallRecord[] = shouldRecordCall
|
||||
? [
|
||||
{
|
||||
id: `call_${behavior.id}`,
|
||||
functionName: behavior.functionName,
|
||||
argumentsJson: mockToolArguments(behavior),
|
||||
resultJson: behavior.mockResponse.body,
|
||||
outcome: behavior.mockResponse.outcome,
|
||||
durationMs: behavior.mockResponse.delayMs + 84,
|
||||
},
|
||||
]
|
||||
: [];
|
||||
const actual = shouldRecordCall
|
||||
? `实际调用 1 次 ${behavior.functionName}`
|
||||
: `实际调用 0 次 ${behavior.functionName}`;
|
||||
const reason = fail
|
||||
? behavior.expectation === "not_called"
|
||||
? "检测到本轮不应发生的工具调用。"
|
||||
: "未检测到满足次数要求的工具调用。"
|
||||
: "";
|
||||
return {
|
||||
evaluation: {
|
||||
id: behavior.id,
|
||||
label: `工具 · ${behavior.functionName}`,
|
||||
kind: "tool_call",
|
||||
status: fail ? "fail" : "pass",
|
||||
expected: toolExpectation(behavior),
|
||||
actual,
|
||||
reason,
|
||||
},
|
||||
toolCalls,
|
||||
};
|
||||
}
|
||||
|
||||
function createCaseResult(
|
||||
item: TestCase,
|
||||
caseIndex: number,
|
||||
maxAttempts: number,
|
||||
): { result: BatchRunCase; plannedError: BatchExecutionError | null } {
|
||||
const plannedError = shouldError(caseIndex);
|
||||
const plannedFailure = !plannedError && shouldFail(caseIndex, item);
|
||||
let failureAssigned = false;
|
||||
const turns = item.turns.map((turn, turnIndex) => {
|
||||
const toolCalls: BatchToolCallRecord[] = [];
|
||||
const evaluations = turn.behaviors.map((behavior) => {
|
||||
const fail = plannedFailure && !failureAssigned;
|
||||
if (fail) failureAssigned = true;
|
||||
const result = createBehaviorEvaluation(behavior, fail);
|
||||
toolCalls.push(...result.toolCalls);
|
||||
return result.evaluation;
|
||||
});
|
||||
return {
|
||||
id: turn.id,
|
||||
index: turnIndex,
|
||||
userInput: turn.userInput,
|
||||
assistantReply: mockAssistantReply(turn.behaviors),
|
||||
toolCalls,
|
||||
evaluations,
|
||||
};
|
||||
});
|
||||
const overallCriteria = item.overallCriteria.map((criterion) => {
|
||||
const fail = plannedFailure && !failureAssigned;
|
||||
if (fail) failureAssigned = true;
|
||||
return {
|
||||
id: criterion.id,
|
||||
label: criterion.name,
|
||||
kind: "overall" as const,
|
||||
status: fail ? ("fail" as const) : ("pass" as const),
|
||||
expected: criterion.criteria,
|
||||
actual: fail
|
||||
? "整段对话未完整达到该业务目标。"
|
||||
: "整段对话达到该业务目标。",
|
||||
reason: fail ? "LLM 判断认为整段对话未满足该项标准。" : "",
|
||||
};
|
||||
});
|
||||
return {
|
||||
result: {
|
||||
id: item.id,
|
||||
name: item.name,
|
||||
status: "waiting",
|
||||
turns,
|
||||
overallCriteria,
|
||||
attemptCount: 0,
|
||||
maxAttempts,
|
||||
executionError: null,
|
||||
},
|
||||
plannedError: plannedError
|
||||
? {
|
||||
code: "PIPELINE_TIMEOUT",
|
||||
message: "等待 pipeline 完成回复时超过单用例超时时间。",
|
||||
stage: "pipeline",
|
||||
retryable: true,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function createMockBatchRun(input: {
|
||||
title: string;
|
||||
assistantName: string;
|
||||
cases: TestCase[];
|
||||
concurrency: number;
|
||||
timeoutSecs: number;
|
||||
failureStrategy: BatchFailureStrategy;
|
||||
errorRetryCount: number;
|
||||
errorStrategy: BatchErrorStrategy;
|
||||
}): {
|
||||
snapshot: BatchRunSnapshot;
|
||||
executionPlan: BatchMockExecutionPlan;
|
||||
} {
|
||||
const preparedCases = input.cases.map((item, index) =>
|
||||
createCaseResult(item, index, input.errorRetryCount + 1),
|
||||
);
|
||||
const executionPlan = Object.fromEntries(
|
||||
preparedCases.flatMap(({ result, plannedError }) =>
|
||||
plannedError ? [[result.id, plannedError]] : [],
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
snapshot: {
|
||||
title: input.title,
|
||||
assistantName: input.assistantName,
|
||||
config: {
|
||||
suiteCount: new Set(input.cases.map((item) => item.suiteId)).size,
|
||||
concurrency: input.concurrency,
|
||||
timeoutSecs: input.timeoutSecs,
|
||||
failureStrategy: input.failureStrategy,
|
||||
errorRetryCount: input.errorRetryCount,
|
||||
errorStrategy: input.errorStrategy,
|
||||
},
|
||||
startedAt: new Date().toISOString(),
|
||||
finishedAt: null,
|
||||
stopReason: null,
|
||||
cases: preparedCases.map(({ result }) => result),
|
||||
},
|
||||
executionPlan,
|
||||
};
|
||||
}
|
||||
|
||||
export function hasEvaluationFailure(item: BatchRunCase): boolean {
|
||||
const evaluations = [
|
||||
...item.turns.flatMap((turn) => turn.evaluations),
|
||||
...item.overallCriteria,
|
||||
];
|
||||
return evaluations.some((evaluation) => evaluation.status === "fail");
|
||||
}
|
||||
|
||||
export function firstEvaluationFailureReason(item: BatchRunCase): string {
|
||||
const evaluations = [
|
||||
...item.turns.flatMap((turn) => turn.evaluations),
|
||||
@@ -358,15 +114,13 @@ export function countByStatus(cases: BatchRunCase[]) {
|
||||
waiting: 0,
|
||||
skipped: 0,
|
||||
};
|
||||
for (const item of cases) {
|
||||
counts[item.status] += 1;
|
||||
}
|
||||
for (const item of cases) counts[item.status] += 1;
|
||||
return counts;
|
||||
}
|
||||
|
||||
export function formatRunTime(iso: string): string {
|
||||
const date = new Date(iso);
|
||||
const pad = (n: number) => String(n).padStart(2, "0");
|
||||
const pad = (value: number) => String(value).padStart(2, "0");
|
||||
return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}`;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* 测试集 / 测试用例 — 管理页用的本地 mock。
|
||||
* 测试集 / 测试用例的前后端共享契约与编辑器辅助函数。
|
||||
* 两层：Test Suite → Test Case。
|
||||
* MVP 仅运行固定文字脚本；其它输入模式保留为产品路线提示。
|
||||
*/
|
||||
@@ -12,7 +12,7 @@ export type TestCaseInputMode =
|
||||
| "user_sim_text"
|
||||
| "user_sim_voice";
|
||||
|
||||
type TestCaseResult = "pass" | "fail" | "not_run";
|
||||
export type TestCaseResult = "pass" | "fail" | "not_run";
|
||||
|
||||
export type AssertionType = "keyword" | "llm";
|
||||
|
||||
@@ -105,8 +105,9 @@ export type TestSuite = {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
/** 关联助手展示名；MVP 直接存文案，接 API 后可改成 assistantId */
|
||||
assistantName: string;
|
||||
caseCount: number;
|
||||
passedCount: number;
|
||||
runCount: number;
|
||||
updatedAt: string;
|
||||
};
|
||||
|
||||
@@ -160,12 +161,60 @@ export function getTestCaseValidationMessage(
|
||||
}
|
||||
if (context.role === "tool_call" || context.role === "tool_result") {
|
||||
try {
|
||||
JSON.parse(context.content);
|
||||
const payload = JSON.parse(context.content);
|
||||
if (
|
||||
context.role === "tool_call" &&
|
||||
(payload === null || Array.isArray(payload) || typeof payload !== "object")
|
||||
) {
|
||||
return `上下文第 ${index + 1} 条 Tool Call 参数必须是 JSON 对象`;
|
||||
}
|
||||
} catch {
|
||||
return `上下文第 ${index + 1} 条工具数据必须是有效 JSON`;
|
||||
}
|
||||
}
|
||||
}
|
||||
const pendingToolCalls: Array<{ id: string | null; toolName: string }> = [];
|
||||
const seenToolCallIds = new Set<string>();
|
||||
for (let index = 0; index < item.contextTurns.length; index += 1) {
|
||||
const context = item.contextTurns[index];
|
||||
if (context.role === "tool_call") {
|
||||
const id = context.toolCallId?.trim() || null;
|
||||
if (id && seenToolCallIds.has(id)) {
|
||||
return `上下文第 ${index + 1} 条 Tool Call ID 重复`;
|
||||
}
|
||||
if (id) seenToolCallIds.add(id);
|
||||
pendingToolCalls.push({ id, toolName: context.toolName?.trim() ?? "" });
|
||||
continue;
|
||||
}
|
||||
if (context.role !== "tool_result") continue;
|
||||
const resultId = context.toolCallId?.trim() || null;
|
||||
let matchedIndex = -1;
|
||||
for (
|
||||
let pendingIndex = pendingToolCalls.length - 1;
|
||||
pendingIndex >= 0;
|
||||
pendingIndex -= 1
|
||||
) {
|
||||
const pending = pendingToolCalls[pendingIndex];
|
||||
if (
|
||||
resultId ? pending.id === resultId : pending.toolName === context.toolName?.trim()
|
||||
) {
|
||||
matchedIndex = pendingIndex;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matchedIndex < 0) {
|
||||
return `上下文第 ${index + 1} 条 Tool Result 没有匹配的 Tool Call`;
|
||||
}
|
||||
const [matched] = pendingToolCalls.splice(matchedIndex, 1);
|
||||
if (matched.toolName !== context.toolName?.trim()) {
|
||||
return `上下文第 ${index + 1} 条 Tool Result 的工具名称不匹配`;
|
||||
}
|
||||
}
|
||||
if (pendingToolCalls.length > 0) {
|
||||
return `上下文 Tool Call 缺少对应的 Tool Result：${pendingToolCalls
|
||||
.map((item) => item.toolName)
|
||||
.join("、")}`;
|
||||
}
|
||||
const emptyTurnIndex = item.turns.findIndex(
|
||||
(turn) => !turn.userInput.trim(),
|
||||
);
|
||||
@@ -311,26 +360,19 @@ export function getExpectedBehaviorValidationMessage(
|
||||
return null;
|
||||
}
|
||||
|
||||
let turnSeq = 1;
|
||||
let behaviorSeq = 1;
|
||||
let criterionSeq = 1;
|
||||
|
||||
function nextTurnId() {
|
||||
const id = `turn_${String(turnSeq).padStart(4, "0")}`;
|
||||
turnSeq += 1;
|
||||
return id;
|
||||
function newEditorId(prefix: string) {
|
||||
const randomPart = globalThis.crypto?.randomUUID
|
||||
? globalThis.crypto.randomUUID().replaceAll("-", "")
|
||||
: `${Date.now().toString(36)}${Math.random().toString(36).slice(2)}`;
|
||||
return `${prefix}_${randomPart}`;
|
||||
}
|
||||
|
||||
function nextBehaviorId() {
|
||||
const id = `beh_${String(behaviorSeq).padStart(4, "0")}`;
|
||||
behaviorSeq += 1;
|
||||
return id;
|
||||
return newEditorId("beh");
|
||||
}
|
||||
|
||||
function nextCriterionId() {
|
||||
const id = `criterion_${String(criterionSeq).padStart(4, "0")}`;
|
||||
criterionSeq += 1;
|
||||
return id;
|
||||
return newEditorId("criterion");
|
||||
}
|
||||
|
||||
export function createOverallCriterion(
|
||||
@@ -365,7 +407,7 @@ export function createEmptyFixedInputTurn(
|
||||
userInput = "",
|
||||
): FixedInputTurn {
|
||||
return {
|
||||
id: nextTurnId(),
|
||||
id: newEditorId("turn"),
|
||||
userInput,
|
||||
behaviors: [],
|
||||
};
|
||||
@@ -435,10 +477,6 @@ function cloneBehavior(behavior: ExpectedBehavior): ExpectedBehavior {
|
||||
};
|
||||
}
|
||||
|
||||
function cloneBehaviorWithNewId(behavior: ExpectedBehavior): ExpectedBehavior {
|
||||
return { ...cloneBehavior(behavior), id: nextBehaviorId() };
|
||||
}
|
||||
|
||||
export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
|
||||
return turns.map((turn) => ({
|
||||
id: turn.id,
|
||||
@@ -447,480 +485,6 @@ export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
|
||||
}));
|
||||
}
|
||||
|
||||
/** 新建用例时复制轮次并分配新 id */
|
||||
function cloneTurnsWithNewIds(turns: FixedInputTurn[]): FixedInputTurn[] {
|
||||
return turns.map((turn) => ({
|
||||
id: nextTurnId(),
|
||||
userInput: turn.userInput,
|
||||
behaviors: turn.behaviors.map(cloneBehaviorWithNewId),
|
||||
}));
|
||||
}
|
||||
|
||||
const INITIAL_SUITES: TestSuite[] = [
|
||||
{
|
||||
id: "suite_001",
|
||||
name: "事故基础流程",
|
||||
description: "核心业务流程和正常事故处理",
|
||||
assistantName: "视频快处助手",
|
||||
updatedAt: "2026-08-05T10:20:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "suite_002",
|
||||
name: "异常输入与澄清",
|
||||
description: "模糊表达、误打断、主动唤醒等",
|
||||
assistantName: "视频快处助手",
|
||||
updatedAt: "2026-08-04T16:40:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "suite_003",
|
||||
name: "实时语音交互",
|
||||
description: "打断、延迟、VAD、语音链路等",
|
||||
assistantName: "视频快处助手",
|
||||
updatedAt: "2026-08-03T09:15:00+08:00",
|
||||
},
|
||||
];
|
||||
|
||||
type RawCaseSeed = Omit<TestCase, "sortOrder" | "inputMode" | "overallCriteria"> & {
|
||||
inputMode?: TestCaseInputMode;
|
||||
overallCriteria?: OverallCriterion[];
|
||||
};
|
||||
|
||||
function seedReplyTurn(
|
||||
seedId: string,
|
||||
userInput: string,
|
||||
expectation: Omit<ReplyExpectedBehavior, "id" | "type">,
|
||||
): FixedInputTurn {
|
||||
return {
|
||||
id: `turn_seed_${seedId}`,
|
||||
userInput,
|
||||
behaviors: [
|
||||
{
|
||||
id: `beh_seed_${seedId}`,
|
||||
type: "reply",
|
||||
...expectation,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const RAW_CASE_SEEDS: RawCaseSeed[] = [
|
||||
{
|
||||
id: "tc_001",
|
||||
suiteId: "suite_001",
|
||||
name: "正常双车事故开场",
|
||||
description: "开场后用户补充事故经过",
|
||||
lastResult: "pass",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("001", "我这里刚刚撞了一下。", {
|
||||
assertionType: "keyword",
|
||||
keywords: ["经过", "描述", "受伤"],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria: "",
|
||||
}),
|
||||
],
|
||||
updatedAt: "2026-08-05T10:18:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "tc_002",
|
||||
suiteId: "suite_001",
|
||||
name: "有人伤转人工",
|
||||
description: "用户提到人伤时应引导转人工",
|
||||
lastResult: "pass",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("002", "有人受伤了，流血不止。", {
|
||||
assertionType: "llm",
|
||||
keywords: [],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria:
|
||||
"Agent 应识别人员受伤风险，明确告知将转接人工，并避免继续推进普通快处流程。",
|
||||
}),
|
||||
],
|
||||
overallCriteria: [
|
||||
{
|
||||
id: "criterion_seed_001",
|
||||
type: "llm",
|
||||
name: "人伤流程正确",
|
||||
criteria:
|
||||
"整段对话应正确识别人员受伤场景，及时并准确转接人工处理。",
|
||||
},
|
||||
{
|
||||
id: "criterion_seed_002",
|
||||
type: "llm",
|
||||
name: "不推进普通快处",
|
||||
criteria:
|
||||
"确认存在人员受伤后，不应继续引导用户进入普通事故快处流程。",
|
||||
},
|
||||
],
|
||||
updatedAt: "2026-08-05T10:12:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "tc_101",
|
||||
suiteId: "suite_002",
|
||||
name: "用户说“喂”",
|
||||
description: "验证主动唤醒回复且业务状态不推进",
|
||||
lastResult: "pass",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("101", "喂", {
|
||||
assertionType: "keyword",
|
||||
keywords: ["我在", "请说", "继续"],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria: "",
|
||||
}),
|
||||
],
|
||||
updatedAt: "2026-08-04T16:35:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "tc_102",
|
||||
suiteId: "suite_002",
|
||||
name: "用户只说“嗯”",
|
||||
description: "短促确认不应误推进流程",
|
||||
lastResult: "fail",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("102", "嗯", {
|
||||
assertionType: "llm",
|
||||
keywords: [],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria:
|
||||
"Agent 应当回应用户的主动唤醒，同时继续引导用户描述事故情况，不应该无响应。",
|
||||
}),
|
||||
],
|
||||
updatedAt: "2026-08-04T16:20:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "tc_103",
|
||||
suiteId: "suite_002",
|
||||
name: "模糊事故描述",
|
||||
description: "地点含糊时应主动澄清",
|
||||
lastResult: "not_run",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("103", "就在那边……撞了一下。", {
|
||||
assertionType: "keyword",
|
||||
keywords: ["哪里", "路口", "路名", "再说"],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria: "",
|
||||
}),
|
||||
],
|
||||
updatedAt: "2026-08-04T15:30:00+08:00",
|
||||
},
|
||||
{
|
||||
id: "tc_201",
|
||||
suiteId: "suite_003",
|
||||
name: "用户打断播报",
|
||||
description: "播报中打断后正确切换聆听并承接",
|
||||
lastResult: "pass",
|
||||
contextTurns: [],
|
||||
turns: [
|
||||
seedReplyTurn("201", "等一下，对方走了。", {
|
||||
assertionType: "llm",
|
||||
keywords: [],
|
||||
keywordMatchMode: "any",
|
||||
negateKeywords: false,
|
||||
llmCriteria:
|
||||
"Agent 应立即停止原播报思路，确认已听到用户新信息，并围绕「对方离开」继续询问。",
|
||||
}),
|
||||
],
|
||||
updatedAt: "2026-08-03T09:10:00+08:00",
|
||||
},
|
||||
];
|
||||
|
||||
/** 按套件出现顺序写入 sortOrder，并补齐可选字段。 */
|
||||
const INITIAL_CASES: TestCase[] = (() => {
|
||||
const counters = new Map<string, number>();
|
||||
return RAW_CASE_SEEDS.map((item) => {
|
||||
const order = counters.get(item.suiteId) ?? 0;
|
||||
counters.set(item.suiteId, order + 1);
|
||||
return {
|
||||
...item,
|
||||
inputMode: item.inputMode ?? DEFAULT_INPUT_MODE,
|
||||
turns: cloneTurns(item.turns),
|
||||
overallCriteria: cloneOverallCriteria(item.overallCriteria),
|
||||
sortOrder: order,
|
||||
};
|
||||
});
|
||||
})();
|
||||
|
||||
/** 会话内可变的 mock 仓库（刷新页面会重置） */
|
||||
let suites = [...INITIAL_SUITES];
|
||||
let cases = structuredClone(INITIAL_CASES);
|
||||
let suiteSeq = 4;
|
||||
let caseSeq = 300;
|
||||
|
||||
function nowIso() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function nextSuiteId() {
|
||||
const id = `suite_${String(suiteSeq).padStart(3, "0")}`;
|
||||
suiteSeq += 1;
|
||||
return id;
|
||||
}
|
||||
|
||||
function nextCaseId() {
|
||||
const id = `tc_${String(caseSeq).padStart(3, "0")}`;
|
||||
caseSeq += 1;
|
||||
return id;
|
||||
}
|
||||
|
||||
export function listTestSuites(): TestSuite[] {
|
||||
return [...suites].sort(
|
||||
(a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime(),
|
||||
);
|
||||
}
|
||||
|
||||
export function getTestSuite(id: string): TestSuite | null {
|
||||
return suites.find((item) => item.id === id) ?? null;
|
||||
}
|
||||
|
||||
function getTestCase(id: string): TestCase | null {
|
||||
return cases.find((item) => item.id === id) ?? null;
|
||||
}
|
||||
|
||||
export function listTestCases(suiteId: string): TestCase[] {
|
||||
return cases
|
||||
.filter((item) => item.suiteId === suiteId)
|
||||
.sort(
|
||||
(a, b) =>
|
||||
a.sortOrder - b.sortOrder || a.name.localeCompare(b.name, "zh-CN"),
|
||||
);
|
||||
}
|
||||
|
||||
export function suiteCaseStats(suiteId: string): {
|
||||
total: number;
|
||||
passed: number;
|
||||
run: number;
|
||||
} {
|
||||
const items = cases.filter((item) => item.suiteId === suiteId);
|
||||
const run = items.filter((item) => item.lastResult !== "not_run");
|
||||
const passed = run.filter((item) => item.lastResult === "pass");
|
||||
return { total: items.length, passed: passed.length, run: run.length };
|
||||
}
|
||||
|
||||
export function createTestSuite(input: {
|
||||
name: string;
|
||||
description: string;
|
||||
assistantName: string;
|
||||
}): TestSuite {
|
||||
const suite: TestSuite = {
|
||||
id: nextSuiteId(),
|
||||
name: input.name.trim(),
|
||||
description: input.description.trim(),
|
||||
assistantName: input.assistantName.trim() || "未关联助手",
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
suites = [suite, ...suites];
|
||||
return suite;
|
||||
}
|
||||
|
||||
export function updateTestSuite(
|
||||
id: string,
|
||||
patch: Partial<Pick<TestSuite, "name" | "description" | "assistantName">>,
|
||||
): TestSuite | null {
|
||||
const index = suites.findIndex((item) => item.id === id);
|
||||
if (index < 0) return null;
|
||||
const next = {
|
||||
...suites[index],
|
||||
...patch,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
suites = [...suites.slice(0, index), next, ...suites.slice(index + 1)];
|
||||
return next;
|
||||
}
|
||||
|
||||
export function removeTestSuite(id: string): boolean {
|
||||
const before = suites.length;
|
||||
suites = suites.filter((item) => item.id !== id);
|
||||
cases = cases.filter((item) => item.suiteId !== id);
|
||||
return suites.length < before;
|
||||
}
|
||||
|
||||
/** 复制测试集及其全部用例；名称加「（副本）」 */
|
||||
export function duplicateTestSuite(id: string): TestSuite | null {
|
||||
const source = getTestSuite(id);
|
||||
if (!source) return null;
|
||||
|
||||
const copied = createTestSuite({
|
||||
name: `${source.name}（副本）`,
|
||||
description: source.description,
|
||||
assistantName: source.assistantName,
|
||||
});
|
||||
|
||||
const sourceCases = listTestCases(id);
|
||||
const clonedCases: TestCase[] = sourceCases.map((item, index) => {
|
||||
const turns = cloneTurnsWithNewIds(item.turns);
|
||||
return {
|
||||
id: nextCaseId(),
|
||||
suiteId: copied.id,
|
||||
name: item.name,
|
||||
description: item.description,
|
||||
inputMode: item.inputMode,
|
||||
lastResult: "not_run" as const,
|
||||
contextTurns: item.contextTurns.map((turn) => ({ ...turn })),
|
||||
turns,
|
||||
overallCriteria: cloneOverallCriteria(item.overallCriteria),
|
||||
sortOrder: index,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
});
|
||||
cases = [...cases, ...clonedCases];
|
||||
return copied;
|
||||
}
|
||||
|
||||
export function createTestCase(input: {
|
||||
suiteId: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
}): TestCase | null {
|
||||
if (!getTestSuite(input.suiteId)) return null;
|
||||
const maxOrder = cases
|
||||
.filter((item) => item.suiteId === input.suiteId)
|
||||
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
|
||||
const turns = [createEmptyFixedInputTurn()];
|
||||
const item: TestCase = {
|
||||
id: nextCaseId(),
|
||||
suiteId: input.suiteId,
|
||||
name: (input.name ?? "未命名用例").trim() || "未命名用例",
|
||||
description: (input.description ?? "").trim(),
|
||||
inputMode: DEFAULT_INPUT_MODE,
|
||||
lastResult: "not_run",
|
||||
contextTurns: [],
|
||||
turns,
|
||||
overallCriteria: [],
|
||||
sortOrder: maxOrder + 1,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
cases = [...cases, item];
|
||||
updateTestSuite(input.suiteId, {});
|
||||
return item;
|
||||
}
|
||||
|
||||
/** 复制单条用例；名称加「（副本）」，排在同套件末尾 */
|
||||
export function duplicateTestCase(id: string): TestCase | null {
|
||||
const source = getTestCase(id);
|
||||
if (!source || !getTestSuite(source.suiteId)) return null;
|
||||
|
||||
const maxOrder = cases
|
||||
.filter((item) => item.suiteId === source.suiteId)
|
||||
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
|
||||
|
||||
const turns = cloneTurnsWithNewIds(source.turns);
|
||||
const copied: TestCase = {
|
||||
id: nextCaseId(),
|
||||
suiteId: source.suiteId,
|
||||
name: `${source.name}（副本）`,
|
||||
description: source.description,
|
||||
inputMode: source.inputMode,
|
||||
lastResult: "not_run",
|
||||
contextTurns: source.contextTurns.map((turn) => ({ ...turn })),
|
||||
turns,
|
||||
overallCriteria: cloneOverallCriteria(source.overallCriteria),
|
||||
sortOrder: maxOrder + 1,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
cases = [...cases, copied];
|
||||
updateTestSuite(source.suiteId, {});
|
||||
return copied;
|
||||
}
|
||||
|
||||
type TestCasePatch = Partial<
|
||||
Pick<
|
||||
TestCase,
|
||||
| "name"
|
||||
| "description"
|
||||
| "inputMode"
|
||||
| "contextTurns"
|
||||
| "turns"
|
||||
| "overallCriteria"
|
||||
| "lastResult"
|
||||
>
|
||||
>;
|
||||
|
||||
export function updateTestCase(
|
||||
id: string,
|
||||
patch: TestCasePatch,
|
||||
): TestCase | null {
|
||||
const index = cases.findIndex((item) => item.id === id);
|
||||
if (index < 0) return null;
|
||||
const current = cases[index];
|
||||
const nextTurns =
|
||||
patch.turns !== undefined
|
||||
? cloneTurns(
|
||||
patch.turns.length > 0 ? patch.turns : [createEmptyFixedInputTurn()],
|
||||
)
|
||||
: current.turns;
|
||||
const nextOverallCriteria =
|
||||
patch.overallCriteria !== undefined
|
||||
? cloneOverallCriteria(patch.overallCriteria)
|
||||
: cloneOverallCriteria(current.overallCriteria);
|
||||
const next: TestCase = {
|
||||
...current,
|
||||
...patch,
|
||||
turns: nextTurns,
|
||||
overallCriteria: nextOverallCriteria,
|
||||
updatedAt: nowIso(),
|
||||
};
|
||||
cases = [...cases.slice(0, index), next, ...cases.slice(index + 1)];
|
||||
updateTestSuite(next.suiteId, {});
|
||||
return next;
|
||||
}
|
||||
|
||||
export function removeTestCase(id: string): boolean {
|
||||
const existing = cases.find((item) => item.id === id);
|
||||
if (!existing) return false;
|
||||
cases = cases.filter((item) => item.id !== id);
|
||||
renumberSortOrder(existing.suiteId);
|
||||
updateTestSuite(existing.suiteId, {});
|
||||
return true;
|
||||
}
|
||||
|
||||
/** 批量删除；返回受影响的 suiteId（若有） */
|
||||
export function removeTestCases(ids: string[]): string | null {
|
||||
const idSet = new Set(ids);
|
||||
const affected = cases.find((item) => idSet.has(item.id));
|
||||
if (!affected) return null;
|
||||
const suiteId = affected.suiteId;
|
||||
cases = cases.filter((item) => !idSet.has(item.id));
|
||||
renumberSortOrder(suiteId);
|
||||
updateTestSuite(suiteId, {});
|
||||
return suiteId;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按给定 id 顺序写回 sortOrder（应包含该套件全部用例 id）。
|
||||
* 用于拖拽结束后持久化顺序。
|
||||
*/
|
||||
export function reorderTestCases(suiteId: string, orderedIds: string[]): void {
|
||||
const orderMap = new Map(orderedIds.map((id, index) => [id, index]));
|
||||
cases = cases.map((item) => {
|
||||
if (item.suiteId !== suiteId) return item;
|
||||
const nextOrder = orderMap.get(item.id);
|
||||
if (nextOrder === undefined) return item;
|
||||
return { ...item, sortOrder: nextOrder };
|
||||
});
|
||||
updateTestSuite(suiteId, {});
|
||||
}
|
||||
|
||||
function renumberSortOrder(suiteId: string) {
|
||||
const ordered = cases
|
||||
.filter((item) => item.suiteId === suiteId)
|
||||
.sort((a, b) => a.sortOrder - b.sortOrder);
|
||||
const orderMap = new Map(ordered.map((item, index) => [item.id, index]));
|
||||
cases = cases.map((item) => {
|
||||
if (item.suiteId !== suiteId) return item;
|
||||
const nextOrder = orderMap.get(item.id);
|
||||
return nextOrder === undefined ? item : { ...item, sortOrder: nextOrder };
|
||||
});
|
||||
}
|
||||
|
||||
export function formatUpdatedAt(value?: string | null) {
|
||||
if (!value) return "—";
|
||||
const date = new Date(value);
|
||||
|
||||
Reference in New Issue
Block a user