Expand test case fixed-input editor and batch case detail split view.

Unify multi-turn behaviors (reply/tool), overall expectation, and input-mode picker while keeping non-text modes as coming soon; batch completed/running use a half-and-half detail panel below the top bar.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Xin Wang
2026-08-09 17:08:53 +08:00
co-authored by Cursor
parent e988f9c133
commit ba6c7c6be3
11 changed files with 2363 additions and 687 deletions
+508 -68
View File
@@ -1,15 +1,19 @@
/**
* 测试集 / 测试用例 — 管理页用的本地 mock。
* 两层:Test Suite → Test Case。
* 第一版只支持「单步回复 / Next Reply Test」:只编辑,不运行。
* 输入模式:固定脚本(文字 / 逐轮语音 / 连续语音)与用户模拟(文字 / 语音)。
*/
export type TestCaseKind =
| "next_reply"
| "tool_call"
| "fixed_dialogue"
| "user_simulation"
| "voice_run";
/** 顶部模式选择器的唯一来源;不再单独维护「测试类型」字段 */
export type TestCaseInputMode =
| "fixed_script_text"
| "fixed_script_turn_voice"
| "fixed_script_continuous_voice"
| "user_sim_text"
| "user_sim_voice";
/** 列表/兼容用粗粒度种类(由 inputMode 推导) */
export type TestCaseKind = "fixed_dialogue" | "user_simulation";
export type TestCaseResult = "pass" | "fail" | "not_run";
@@ -22,6 +26,80 @@ export type ContextTurn = {
content: string;
};
export type VoiceSettings = {
voiceId: string;
/** 语速倍率,如 1.0 */
speed: number;
};
/** 连续语音模式下每轮的发送时机 */
export type TurnSendTiming =
| "after_previous_reply"
| "after_agent_starts"
| "fixed_delay";
export type UserSimulationConfig = {
role: string;
goal: string;
knownFacts: string;
behaviorNotes: string;
maxTurns: number;
};
/** 工具参数校验方式(未配置 = 不参与断言) */
export type ToolParamMatchMode = "exact" | "regex" | "llm";
export type ToolParamAssertion = {
name: string;
matchMode: ToolParamMatchMode;
value: string;
};
/** 预期行为:回复要求 */
export type ReplyExpectedBehavior = {
id: string;
type: "reply";
assertionType: AssertionType;
keywords: string[];
keywordMatchMode: KeywordMatchMode;
llmCriteria: string;
};
/** 预期行为:工具调用(至少调用一次指定工具) */
export type ToolCallExpectedBehavior = {
id: string;
type: "tool_call";
toolId: string;
functionName: string;
/** 仅包含用户主动配置的参数;未列出的参数不校验 */
paramAssertions: ToolParamAssertion[];
};
export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
/**
* 固定输入的一轮:只编辑 User 输入;Agent 回复 / Tool Call 由运行时生成。
* behaviors 为 0~N 个预期行为(AND);空数组表示仅推动对话、不做断言。
*/
export type FixedInputTurn = {
id: string;
userInput: string;
behaviors: ExpectedBehavior[];
/** 连续语音:发送时机(其它模式可忽略) */
sendTiming?: TurnSendTiming;
/** 连续语音:延迟毫秒(仅部分时机需要) */
sendDelayMs?: number;
};
/**
* 整段对话结束后的业务目标断言(可选,独立于每轮 behaviors)。
* MVP 仅支持 LLM 判断。
*/
export type OverallExpectation = {
type: "llm";
criteria: string;
};
export type TestSuite = {
id: string;
name: string;
@@ -36,50 +114,322 @@ export type TestCase = {
suiteId: string;
name: string;
description: string;
/** 输入模式(编辑器顶部选择器) */
inputMode: TestCaseInputMode;
/** 由 inputMode 推导,供列表筛选等兼容读取 */
kind: TestCaseKind;
lastResult: TestCaseResult;
/** 对话上下文(通常是 Agent 上一轮) */
contextTurns: ContextTurn[];
/** 当前用户输入 */
/** 固定脚本轮次 */
turns: FixedInputTurn[];
/** 语音相关模式的音色 / 语速 */
voiceSettings?: VoiceSettings | null;
/** 用户模拟配置 */
userSimulation?: UserSimulationConfig | null;
/** 整段对话业务目标预期(可选) */
overallExpectation?: OverallExpectation | null;
/**
* 以下字段由 turns[0] 同步,供批量测试等旧逻辑读取。
* 编辑与保存以 turns 为准。
*/
userInput: string;
assertionType: AssertionType;
keywords: string[];
keywordMatchMode: KeywordMatchMode;
/** LLM 判断时的判断标准 */
llmCriteria: string;
/** 套件内排序,越小越靠前 */
sortOrder: number;
updatedAt: string;
};
/** 第一版仅「单步回复」可编辑,其余类型显示开发中 */
export const SUPPORTED_TEST_CASE_KIND: TestCaseKind = "next_reply";
export const DEFAULT_INPUT_MODE: TestCaseInputMode = "fixed_script_text";
export const TEST_CASE_KIND_LABEL: Record<TestCaseKind, string> = {
next_reply: "单步回复",
tool_call: "工具调用",
fixed_dialogue: "固定对话",
user_simulation: "用户模拟",
voice_run: "语音运行",
export function kindFromInputMode(mode: TestCaseInputMode): TestCaseKind {
return mode === "user_sim_text" || mode === "user_sim_voice"
? "user_simulation"
: "fixed_dialogue";
}
export function isFixedScriptMode(mode: TestCaseInputMode): boolean {
return (
mode === "fixed_script_text" ||
mode === "fixed_script_turn_voice" ||
mode === "fixed_script_continuous_voice"
);
}
export function isVoiceMode(mode: TestCaseInputMode): boolean {
return (
mode === "fixed_script_turn_voice" ||
mode === "fixed_script_continuous_voice" ||
mode === "user_sim_voice"
);
}
export const TEST_CASE_INPUT_MODE_LABEL: Record<TestCaseInputMode, string> = {
fixed_script_text: "固定脚本 · 文字",
fixed_script_turn_voice: "固定脚本 · 逐轮语音",
fixed_script_continuous_voice: "固定脚本 · 连续语音",
user_sim_text: "用户模拟 · 文字",
user_sim_voice: "用户模拟 · 语音",
};
export const TEST_CASE_KIND_OPTIONS: {
value: TestCaseKind;
label: string;
available: boolean;
}[] = [
{ value: "next_reply", label: "单步回复", available: true },
{ value: "tool_call", label: "工具调用", available: false },
{ value: "fixed_dialogue", label: "固定对话", available: false },
{ value: "user_simulation", label: "用户模拟", available: false },
{ value: "voice_run", label: "语音运行", available: false },
];
/** 列表徽章用短标签 */
export const TEST_CASE_INPUT_MODE_SHORT_LABEL: Record<
TestCaseInputMode,
string
> = {
fixed_script_text: "固定文字",
fixed_script_turn_voice: "逐轮语音",
fixed_script_continuous_voice: "连续语音",
user_sim_text: "模拟文字",
user_sim_voice: "模拟语音",
};
/** @deprecated 使用 TEST_CASE_INPUT_MODE_LABEL;保留给旧引用 */
export const TEST_CASE_KIND_LABEL: Record<TestCaseKind, string> = {
fixed_dialogue: "固定脚本",
user_simulation: "用户模拟",
};
export const VOICE_OPTIONS = [
{ value: "zh_female", label: "普通话女声" },
{ value: "zh_male", label: "普通话男声" },
] as const;
export const VOICE_SPEED_OPTIONS = [
{ value: "0.8", label: "0.8x" },
{ value: "1.0", label: "1.0x" },
{ value: "1.2", label: "1.2x" },
{ value: "1.5", label: "1.5x" },
] as const;
export const TURN_SEND_TIMING_LABEL: Record<TurnSendTiming, string> = {
after_previous_reply: "上一轮回复结束后",
after_agent_starts: "Agent 开始回复后",
fixed_delay: "固定延迟",
};
export function createDefaultVoiceSettings(): VoiceSettings {
return { voiceId: "zh_female", speed: 1.0 };
}
export function createDefaultUserSimulation(): UserSimulationConfig {
return {
role: "",
goal: "",
knownFacts: "",
behaviorNotes: "",
maxTurns: 10,
};
}
export function cloneVoiceSettings(
value: VoiceSettings | null | undefined,
): VoiceSettings | null {
if (!value) return null;
return { voiceId: value.voiceId, speed: value.speed };
}
export function cloneUserSimulation(
value: UserSimulationConfig | null | undefined,
): UserSimulationConfig | null {
if (!value) return null;
return {
role: value.role,
goal: value.goal,
knownFacts: value.knownFacts,
behaviorNotes: value.behaviorNotes,
maxTurns: value.maxTurns,
};
}
export const ASSERTION_TYPE_LABEL: Record<AssertionType, string> = {
keyword: "关键词",
llm: "LLM 判断",
};
export const KEYWORD_MATCH_MODE_LABEL: Record<KeywordMatchMode, string> = {
any: "任一匹配",
all: "全部匹配",
};
export const TOOL_PARAM_MATCH_MODE_LABEL: Record<ToolParamMatchMode, string> = {
exact: "精确匹配",
regex: "正则匹配",
llm: "LLM 判断",
};
export const OVERALL_EXPECTATION_CRITERIA_MAX = 2000;
/** 空文本视为未配置 */
export function normalizeOverallExpectation(
criteria: string | null | undefined,
): OverallExpectation | null {
const trimmed = (criteria ?? "").trim();
if (!trimmed) return null;
return {
type: "llm",
criteria: trimmed.slice(0, OVERALL_EXPECTATION_CRITERIA_MAX),
};
}
export function cloneOverallExpectation(
value: OverallExpectation | null | undefined,
): OverallExpectation | null {
if (!value?.criteria.trim()) return null;
return { type: "llm", criteria: value.criteria };
}
let turnSeq = 1;
let behaviorSeq = 1;
function nextTurnId() {
const id = `turn_${String(turnSeq).padStart(4, "0")}`;
turnSeq += 1;
return id;
}
function nextBehaviorId() {
const id = `beh_${String(behaviorSeq).padStart(4, "0")}`;
behaviorSeq += 1;
return id;
}
export function createEmptyFixedInputTurn(
userInput = "",
): FixedInputTurn {
return {
id: nextTurnId(),
userInput,
behaviors: [],
sendTiming: "after_previous_reply",
sendDelayMs: 500,
};
}
export function createReplyBehavior(
assertionType: AssertionType = "llm",
): ReplyExpectedBehavior {
return {
id: nextBehaviorId(),
type: "reply",
assertionType,
keywords: [],
keywordMatchMode: "any",
llmCriteria: "",
};
}
export function createToolCallBehavior(input?: {
toolId?: string;
functionName?: string;
}): ToolCallExpectedBehavior {
return {
id: nextBehaviorId(),
type: "tool_call",
toolId: input?.toolId ?? "",
functionName: input?.functionName ?? "",
paramAssertions: [],
};
}
function cloneBehavior(behavior: ExpectedBehavior): ExpectedBehavior {
if (behavior.type === "reply") {
return {
id: behavior.id,
type: "reply",
assertionType: behavior.assertionType,
keywords: [...behavior.keywords],
keywordMatchMode: behavior.keywordMatchMode,
llmCriteria: behavior.llmCriteria,
};
}
return {
id: behavior.id,
type: "tool_call",
toolId: behavior.toolId,
functionName: behavior.functionName,
paramAssertions: behavior.paramAssertions.map((item) => ({ ...item })),
};
}
function cloneBehaviorWithNewId(behavior: ExpectedBehavior): ExpectedBehavior {
return { ...cloneBehavior(behavior), id: nextBehaviorId() };
}
function firstReplyBehavior(
turns: FixedInputTurn[],
): ReplyExpectedBehavior | null {
for (const turn of turns) {
for (const behavior of turn.behaviors) {
if (behavior.type === "reply") return behavior;
}
}
return null;
}
/** 从首轮回复预期同步旧字段,供批量测试等读取 */
export function legacyFieldsFromTurns(turns: FixedInputTurn[]) {
const first = turns[0];
const reply = firstReplyBehavior(turns);
return {
userInput: first?.userInput ?? "",
assertionType: reply?.assertionType ?? ("keyword" as AssertionType),
keywords: reply ? [...reply.keywords] : ([] as string[]),
keywordMatchMode: reply?.keywordMatchMode ?? ("any" as KeywordMatchMode),
llmCriteria: reply?.llmCriteria ?? "",
};
}
export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: turn.id,
userInput: turn.userInput,
behaviors: turn.behaviors.map(cloneBehavior),
sendTiming: turn.sendTiming ?? "after_previous_reply",
sendDelayMs: turn.sendDelayMs ?? 500,
}));
}
/** 新建用例时复制轮次并分配新 id */
function cloneTurnsWithNewIds(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: nextTurnId(),
userInput: turn.userInput,
behaviors: turn.behaviors.map(cloneBehaviorWithNewId),
sendTiming: turn.sendTiming ?? "after_previous_reply",
sendDelayMs: turn.sendDelayMs ?? 500,
}));
}
/** 从旧的单轮字段构造一轮(含一条回复预期) */
function turnFromLegacyFields(input: {
userInput: string;
assertionType: AssertionType;
keywords: string[];
keywordMatchMode: KeywordMatchMode;
llmCriteria: string;
}): FixedInputTurn {
return {
id: nextTurnId(),
userInput: input.userInput,
behaviors: [
{
id: nextBehaviorId(),
type: "reply",
assertionType: input.assertionType,
keywords: [...input.keywords],
keywordMatchMode: input.keywordMatchMode,
llmCriteria: input.llmCriteria,
},
],
sendTiming: "after_previous_reply",
sendDelayMs: 500,
};
}
const INITIAL_SUITES: TestSuite[] = [
{
id: "suite_001",
@@ -104,13 +454,23 @@ const INITIAL_SUITES: TestSuite[] = [
},
];
const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
type RawCaseSeed = Omit<
TestCase,
"sortOrder" | "turns" | "inputMode" | "voiceSettings" | "userSimulation"
> & {
turns?: FixedInputTurn[];
inputMode?: TestCaseInputMode;
voiceSettings?: VoiceSettings | null;
userSimulation?: UserSimulationConfig | null;
};
const RAW_CASE_SEEDS: RawCaseSeed[] = [
{
id: "tc_001",
suiteId: "suite_001",
name: "正常双车事故开场",
description: "开场后用户补充事故经过",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "pass",
contextTurns: [],
userInput: "我这里刚刚撞了一下。",
@@ -125,7 +485,7 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
suiteId: "suite_001",
name: "有人伤转人工",
description: "用户提到人伤时应引导转人工",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "pass",
contextTurns: [],
userInput: "有人受伤了,流血不止。",
@@ -134,6 +494,11 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
keywordMatchMode: "any",
llmCriteria:
"Agent 应识别人员受伤风险,明确告知将转接人工,并避免继续推进普通快处流程。",
overallExpectation: {
type: "llm",
criteria:
"整段对话应正确识别人员受伤场景,及时并准确转接人工处理,不应继续引导用户进入普通事故快处流程。",
},
updatedAt: "2026-08-05T10:12:00+08:00",
},
{
@@ -141,7 +506,7 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
suiteId: "suite_002",
name: "用户说“喂”",
description: "验证主动唤醒回复且业务状态不推进",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "pass",
contextTurns: [],
userInput: "喂",
@@ -156,7 +521,7 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
suiteId: "suite_002",
name: "用户只说“嗯”",
description: "短促确认不应误推进流程",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "fail",
contextTurns: [],
userInput: "嗯",
@@ -172,7 +537,7 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
suiteId: "suite_002",
name: "模糊事故描述",
description: "地点含糊时应主动澄清",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "not_run",
contextTurns: [],
userInput: "就在那边……撞了一下。",
@@ -187,7 +552,7 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
suiteId: "suite_003",
name: "用户打断播报",
description: "播报中打断后正确切换聆听并承接",
kind: "next_reply",
kind: "fixed_dialogue",
lastResult: "pass",
contextTurns: [],
userInput: "等一下,对方走了。",
@@ -200,13 +565,37 @@ const RAW_CASES: Omit<TestCase, "sortOrder">[] = [
},
];
/** 按套件出现顺序写入 sortOrder */
/** 按套件出现顺序写入 sortOrder,并补齐 turns */
const INITIAL_CASES: TestCase[] = (() => {
const counters = new Map<string, number>();
return RAW_CASES.map((item) => {
return RAW_CASE_SEEDS.map((item) => {
const order = counters.get(item.suiteId) ?? 0;
counters.set(item.suiteId, order + 1);
return { ...item, sortOrder: order };
const turns =
item.turns && item.turns.length > 0
? cloneTurns(item.turns)
: [
turnFromLegacyFields({
userInput: item.userInput,
assertionType: item.assertionType,
keywords: item.keywords,
keywordMatchMode: item.keywordMatchMode,
llmCriteria: item.llmCriteria,
}),
];
const legacy = legacyFieldsFromTurns(turns);
const inputMode = item.inputMode ?? DEFAULT_INPUT_MODE;
return {
...item,
inputMode,
kind: kindFromInputMode(inputMode),
turns,
voiceSettings: cloneVoiceSettings(item.voiceSettings),
userSimulation: cloneUserSimulation(item.userSimulation),
overallExpectation: cloneOverallExpectation(item.overallExpectation),
...legacy,
sortOrder: order,
};
});
})();
@@ -316,22 +705,27 @@ export function duplicateTestSuite(id: string): TestSuite | null {
});
const sourceCases = listTestCases(id);
const clonedCases: TestCase[] = sourceCases.map((item, index) => ({
id: nextCaseId(),
suiteId: copied.id,
name: item.name,
description: item.description,
kind: item.kind,
lastResult: "not_run",
contextTurns: item.contextTurns.map((turn) => ({ ...turn })),
userInput: item.userInput,
assertionType: item.assertionType,
keywords: [...item.keywords],
keywordMatchMode: item.keywordMatchMode,
llmCriteria: item.llmCriteria,
sortOrder: index,
updatedAt: nowIso(),
}));
const clonedCases: TestCase[] = sourceCases.map((item, index) => {
const turns = cloneTurnsWithNewIds(item.turns);
const legacy = legacyFieldsFromTurns(turns);
return {
id: nextCaseId(),
suiteId: copied.id,
name: item.name,
description: item.description,
inputMode: item.inputMode,
kind: kindFromInputMode(item.inputMode),
lastResult: "not_run" as const,
contextTurns: item.contextTurns.map((turn) => ({ ...turn })),
turns,
voiceSettings: cloneVoiceSettings(item.voiceSettings),
userSimulation: cloneUserSimulation(item.userSimulation),
overallExpectation: cloneOverallExpectation(item.overallExpectation),
...legacy,
sortOrder: index,
updatedAt: nowIso(),
};
});
cases = [...cases, ...clonedCases];
return copied;
}
@@ -345,19 +739,22 @@ export function createTestCase(input: {
const maxOrder = cases
.filter((item) => item.suiteId === input.suiteId)
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
const turns = [createEmptyFixedInputTurn()];
const legacy = legacyFieldsFromTurns(turns);
const item: TestCase = {
id: nextCaseId(),
suiteId: input.suiteId,
name: (input.name ?? "未命名用例").trim() || "未命名用例",
description: (input.description ?? "").trim(),
kind: "next_reply",
inputMode: DEFAULT_INPUT_MODE,
kind: kindFromInputMode(DEFAULT_INPUT_MODE),
lastResult: "not_run",
contextTurns: [],
userInput: "",
assertionType: "keyword",
keywords: [],
keywordMatchMode: "any",
llmCriteria: "",
turns,
voiceSettings: null,
userSimulation: null,
overallExpectation: null,
...legacy,
sortOrder: maxOrder + 1,
updatedAt: nowIso(),
};
@@ -375,19 +772,22 @@ export function duplicateTestCase(id: string): TestCase | null {
.filter((item) => item.suiteId === source.suiteId)
.reduce((max, item) => Math.max(max, item.sortOrder), -1);
const turns = cloneTurnsWithNewIds(source.turns);
const legacy = legacyFieldsFromTurns(turns);
const copied: TestCase = {
id: nextCaseId(),
suiteId: source.suiteId,
name: `${source.name}(副本)`,
description: source.description,
kind: source.kind,
inputMode: source.inputMode,
kind: kindFromInputMode(source.inputMode),
lastResult: "not_run",
contextTurns: source.contextTurns.map((turn) => ({ ...turn })),
userInput: source.userInput,
assertionType: source.assertionType,
keywords: [...source.keywords],
keywordMatchMode: source.keywordMatchMode,
llmCriteria: source.llmCriteria,
turns,
voiceSettings: cloneVoiceSettings(source.voiceSettings),
userSimulation: cloneUserSimulation(source.userSimulation),
overallExpectation: cloneOverallExpectation(source.overallExpectation),
...legacy,
sortOrder: maxOrder + 1,
updatedAt: nowIso(),
};
@@ -401,8 +801,13 @@ export type TestCasePatch = Partial<
TestCase,
| "name"
| "description"
| "inputMode"
| "kind"
| "contextTurns"
| "turns"
| "voiceSettings"
| "userSimulation"
| "overallExpectation"
| "userInput"
| "assertionType"
| "keywords"
@@ -418,9 +823,44 @@ export function updateTestCase(
): TestCase | null {
const index = cases.findIndex((item) => item.id === id);
if (index < 0) return null;
const next = {
...cases[index],
const current = cases[index];
const nextTurns =
patch.turns !== undefined
? cloneTurns(
patch.turns.length > 0 ? patch.turns : [createEmptyFixedInputTurn()],
)
: current.turns;
const legacy =
patch.turns !== undefined
? legacyFieldsFromTurns(nextTurns)
: {
userInput: patch.userInput ?? current.userInput,
assertionType: patch.assertionType ?? current.assertionType,
keywords: patch.keywords ?? current.keywords,
keywordMatchMode: patch.keywordMatchMode ?? current.keywordMatchMode,
llmCriteria: patch.llmCriteria ?? current.llmCriteria,
};
const nextOverall =
patch.overallExpectation !== undefined
? cloneOverallExpectation(patch.overallExpectation)
: cloneOverallExpectation(current.overallExpectation);
const nextMode = patch.inputMode ?? current.inputMode;
const next: TestCase = {
...current,
...patch,
inputMode: nextMode,
kind: kindFromInputMode(nextMode),
turns: nextTurns,
voiceSettings:
patch.voiceSettings !== undefined
? cloneVoiceSettings(patch.voiceSettings)
: cloneVoiceSettings(current.voiceSettings),
userSimulation:
patch.userSimulation !== undefined
? cloneUserSimulation(patch.userSimulation)
: cloneUserSimulation(current.userSimulation),
overallExpectation: nextOverall,
...legacy,
updatedAt: nowIso(),
};
cases = [...cases.slice(0, index), next, ...cases.slice(index + 1)];