1274 lines
38 KiB
TypeScript
1274 lines
38 KiB
TypeScript
"use client";
|
|
|
|
/**
|
|
* Debug Drawer 的真实自动测试客户端。
|
|
*
|
|
* 它复用批量测试的持久化 Test Run API，但每次只运行一个用例。这样 Drawer
|
|
* 与批量测试共享同一套 Pipeline、工具 Mock、评估器和运行记录，不维护第二套
|
|
* 测试语义。
|
|
*/
|
|
|
|
import type React from "react";
|
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
|
import {
|
|
ArrowDown,
|
|
Check,
|
|
ChevronDown,
|
|
FlaskConical,
|
|
Loader2,
|
|
Play,
|
|
Square,
|
|
TriangleAlert,
|
|
Wrench,
|
|
X,
|
|
} from "lucide-react";
|
|
|
|
import {
|
|
DebugTimeline,
|
|
type DebugTimelineItem,
|
|
} from "@/components/debug-timeline";
|
|
import { Badge } from "@/components/ui/badge";
|
|
import { Button } from "@/components/ui/button";
|
|
import {
|
|
Popover,
|
|
PopoverContent,
|
|
PopoverTrigger,
|
|
} from "@/components/ui/popover";
|
|
import { SearchInput } from "@/components/ui/search-input";
|
|
import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs";
|
|
import type {
|
|
BatchEvaluationResult,
|
|
BatchRunCase,
|
|
BatchRunSnapshot,
|
|
BatchStartupResult,
|
|
} from "@/data/batch-run";
|
|
import {
|
|
getTestCaseValidationMessage,
|
|
TEST_CASE_INPUT_MODE_SHORT_LABEL,
|
|
type TestCase,
|
|
type TestSuite,
|
|
} from "@/data/test-suites";
|
|
import {
|
|
batchRunsApi,
|
|
modelResourcesApi,
|
|
testAssetsApi,
|
|
testCasesApi,
|
|
testSuitesApi,
|
|
type ModelResource,
|
|
} from "@/lib/api";
|
|
import { cn } from "@/lib/utils";
|
|
|
|
const RUN_POLL_MS = 1000;
|
|
const AUTO_FOLLOW_BOTTOM_THRESHOLD_PX = 32;
|
|
|
|
export type AutoTestPhase =
|
|
| "idle"
|
|
| "ready"
|
|
| "starting"
|
|
| "running"
|
|
| "passed"
|
|
| "failed"
|
|
| "error"
|
|
| "cancelled";
|
|
|
|
export type AutoTestRunState = {
|
|
phase: AutoTestPhase;
|
|
selectedId: string | null;
|
|
currentTurn: number;
|
|
maxTurns: number;
|
|
run: BatchRunSnapshot | null;
|
|
runCase: BatchRunCase | null;
|
|
error: string;
|
|
stopping: boolean;
|
|
blockedMessage: string;
|
|
evaluatorName: string;
|
|
};
|
|
|
|
type AutoTestRunnerOptions = {
|
|
assistantId: string | null;
|
|
enabled: boolean;
|
|
hasUnsavedChanges: boolean;
|
|
dynamicVariables: Record<string, string | number | boolean>;
|
|
dynamicVariablesError: string;
|
|
};
|
|
|
|
function isRunFinished(run: BatchRunSnapshot): boolean {
|
|
return (
|
|
Boolean(run.finishedAt) ||
|
|
run.status === "completed" ||
|
|
run.status === "cancelled"
|
|
);
|
|
}
|
|
|
|
function runnableMessage(item: TestCase): string {
|
|
if (item.inputMode !== "fixed_script_text") {
|
|
return "当前自动测试仅支持固定脚本 · 多模态";
|
|
}
|
|
return getTestCaseValidationMessage(item) ?? "";
|
|
}
|
|
|
|
// ─── Mode tabs ──────────────────────────────────────────────────────────────
|
|
|
|
export function DebugModeTabs({
|
|
mode,
|
|
onChange,
|
|
}: {
|
|
mode: "manual" | "auto";
|
|
onChange: (mode: "manual" | "auto") => void;
|
|
}) {
|
|
return (
|
|
<Tabs
|
|
value={mode}
|
|
onValueChange={(value) => onChange(value as "manual" | "auto")}
|
|
className="w-full gap-0"
|
|
>
|
|
<TabsList className="grid w-full grid-cols-2 bg-surface-strong">
|
|
<TabsTrigger value="manual">手动测试</TabsTrigger>
|
|
<TabsTrigger value="auto">自动测试</TabsTrigger>
|
|
</TabsList>
|
|
</Tabs>
|
|
);
|
|
}
|
|
|
|
// ─── Case picker ────────────────────────────────────────────────────────────
|
|
|
|
function CasePicker({
|
|
suites,
|
|
cases,
|
|
selectedId,
|
|
loading,
|
|
onSelect,
|
|
trigger,
|
|
}: {
|
|
suites: TestSuite[];
|
|
cases: TestCase[];
|
|
selectedId: string | null;
|
|
loading: boolean;
|
|
onSelect: (id: string) => void;
|
|
trigger: React.ReactNode;
|
|
}) {
|
|
const [open, setOpen] = useState(false);
|
|
const [query, setQuery] = useState("");
|
|
const normalizedQuery = query.trim().toLocaleLowerCase();
|
|
|
|
const visibleSuites = useMemo(
|
|
() =>
|
|
suites
|
|
.map((suite) => ({
|
|
suite,
|
|
cases: cases.filter((item) => item.suiteId === suite.id),
|
|
}))
|
|
.map((group) => ({
|
|
...group,
|
|
cases: group.cases.filter((item) => {
|
|
if (!normalizedQuery) return true;
|
|
return [
|
|
item.name,
|
|
item.description,
|
|
TEST_CASE_INPUT_MODE_SHORT_LABEL[item.inputMode],
|
|
].some((value) => value.toLocaleLowerCase().includes(normalizedQuery));
|
|
}),
|
|
}))
|
|
.filter((group) => group.cases.length > 0),
|
|
[cases, normalizedQuery, suites],
|
|
);
|
|
|
|
return (
|
|
<Popover
|
|
open={open}
|
|
onOpenChange={(next) => {
|
|
setOpen(next);
|
|
if (!next) setQuery("");
|
|
}}
|
|
>
|
|
<PopoverTrigger asChild>{trigger}</PopoverTrigger>
|
|
<PopoverContent
|
|
align="start"
|
|
side="top"
|
|
className="w-[min(380px,calc(100vw-2rem))] space-y-3 rounded-2xl p-3"
|
|
>
|
|
<div className="px-1 text-sm font-medium text-foreground">
|
|
选择测试用例
|
|
</div>
|
|
<SearchInput
|
|
value={query}
|
|
onChange={setQuery}
|
|
placeholder="搜索测试用例"
|
|
className="w-full"
|
|
/>
|
|
<div className="scrollbar-subtle max-h-72 space-y-3 overflow-y-auto pr-0.5">
|
|
{loading ? (
|
|
<div className="flex items-center justify-center gap-2 py-8 text-xs text-muted-foreground">
|
|
<Loader2 size={14} className="animate-spin" />
|
|
加载真实测试用例…
|
|
</div>
|
|
) : visibleSuites.length === 0 ? (
|
|
<div className="rounded-xl border border-dashed border-hairline-strong px-3 py-6 text-center text-xs text-muted-foreground">
|
|
{cases.length === 0 ? "还没有测试用例" : "没有匹配的测试用例"}
|
|
</div>
|
|
) : (
|
|
visibleSuites.map(({ suite, cases: suiteCases }) => (
|
|
<section key={suite.id} className="space-y-1">
|
|
<div className="flex items-center justify-between px-2 text-[11px] font-medium text-muted-soft">
|
|
<span className="truncate">{suite.name}</span>
|
|
<span className="tabular-nums">{suiteCases.length}</span>
|
|
</div>
|
|
{suiteCases.map((item) => {
|
|
const unsupported = Boolean(runnableMessage(item));
|
|
return (
|
|
<button
|
|
key={item.id}
|
|
type="button"
|
|
onClick={() => {
|
|
onSelect(item.id);
|
|
setOpen(false);
|
|
}}
|
|
className={cn(
|
|
"flex w-full items-center gap-2 rounded-xl px-2.5 py-2 text-left transition-colors",
|
|
item.id === selectedId
|
|
? "bg-surface-strong"
|
|
: "hover:bg-canvas-soft",
|
|
)}
|
|
>
|
|
<span className="min-w-0 flex-1 truncate text-sm text-foreground">
|
|
{item.name}
|
|
</span>
|
|
<Badge
|
|
variant="secondary"
|
|
className={cn(
|
|
"shrink-0 rounded-full px-2 py-0 text-[10px] font-medium",
|
|
unsupported && "text-muted-soft",
|
|
)}
|
|
>
|
|
{TEST_CASE_INPUT_MODE_SHORT_LABEL[item.inputMode]}
|
|
</Badge>
|
|
</button>
|
|
);
|
|
})}
|
|
</section>
|
|
))
|
|
)}
|
|
</div>
|
|
</PopoverContent>
|
|
</Popover>
|
|
);
|
|
}
|
|
|
|
// ─── Result panel ───────────────────────────────────────────────────────────
|
|
|
|
function AutoFollowViewport({
|
|
children,
|
|
followKey,
|
|
}: {
|
|
children: React.ReactNode;
|
|
followKey: string;
|
|
}) {
|
|
const scrollRef = useRef<HTMLDivElement>(null);
|
|
const contentRef = useRef<HTMLDivElement>(null);
|
|
const isFollowingRef = useRef(true);
|
|
const [isFollowing, setIsFollowing] = useState(true);
|
|
|
|
const updateFollowing = useCallback((next: boolean) => {
|
|
isFollowingRef.current = next;
|
|
setIsFollowing((current) => (current === next ? current : next));
|
|
}, []);
|
|
|
|
const scrollToLatest = useCallback(() => {
|
|
const viewport = scrollRef.current;
|
|
if (!viewport) return;
|
|
viewport.scrollTop = viewport.scrollHeight;
|
|
}, []);
|
|
|
|
const handleScroll = useCallback(() => {
|
|
const viewport = scrollRef.current;
|
|
if (!viewport) return;
|
|
const distanceFromBottom =
|
|
viewport.scrollHeight - viewport.scrollTop - viewport.clientHeight;
|
|
updateFollowing(distanceFromBottom <= AUTO_FOLLOW_BOTTOM_THRESHOLD_PX);
|
|
}, [updateFollowing]);
|
|
|
|
useEffect(() => {
|
|
if (!isFollowingRef.current) return;
|
|
const frame = window.requestAnimationFrame(scrollToLatest);
|
|
return () => window.cancelAnimationFrame(frame);
|
|
}, [followKey, scrollToLatest]);
|
|
|
|
// Runtime cards and images can grow after the test snapshot is rendered.
|
|
useEffect(() => {
|
|
const content = contentRef.current;
|
|
if (!content) return;
|
|
const observer = new ResizeObserver(() => {
|
|
if (isFollowingRef.current) scrollToLatest();
|
|
});
|
|
observer.observe(content);
|
|
return () => observer.disconnect();
|
|
}, [scrollToLatest]);
|
|
|
|
const resumeFollowing = useCallback(() => {
|
|
updateFollowing(true);
|
|
scrollToLatest();
|
|
}, [scrollToLatest, updateFollowing]);
|
|
|
|
return (
|
|
<div className="relative flex min-h-0 flex-1">
|
|
<div
|
|
ref={scrollRef}
|
|
onScroll={handleScroll}
|
|
className="scrollbar-subtle min-h-0 flex-1 overflow-y-auto px-5 py-4"
|
|
>
|
|
<div ref={contentRef}>{children}</div>
|
|
</div>
|
|
{!isFollowing && (
|
|
<Button
|
|
type="button"
|
|
size="sm"
|
|
variant="secondary"
|
|
onClick={resumeFollowing}
|
|
className="absolute bottom-4 left-1/2 h-8 -translate-x-1/2 gap-1.5 rounded-full border border-hairline bg-background/95 px-3 text-xs shadow-md backdrop-blur"
|
|
>
|
|
<ArrowDown size={13} />
|
|
回到最新
|
|
</Button>
|
|
)}
|
|
</div>
|
|
);
|
|
}
|
|
|
|
export function AutoTestLivePanel({
|
|
testCase,
|
|
state,
|
|
onStart,
|
|
}: {
|
|
testCase: TestCase | null;
|
|
state: AutoTestRunState;
|
|
onStart: () => void;
|
|
}) {
|
|
if (!testCase) {
|
|
return (
|
|
<div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-8 text-center">
|
|
<span className="flex h-10 w-10 items-center justify-center rounded-full bg-surface-strong text-muted-soft">
|
|
<FlaskConical size={19} />
|
|
</span>
|
|
<div className="mt-3 text-sm font-medium text-foreground">
|
|
选择测试用例后开始
|
|
</div>
|
|
<Button
|
|
type="button"
|
|
variant="secondary"
|
|
className="mt-4 h-10 gap-2 rounded-full px-5"
|
|
disabled
|
|
title="请先在下方选择测试用例"
|
|
>
|
|
<Play size={14} />
|
|
开始测试
|
|
</Button>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
if (state.phase === "idle" || state.phase === "ready") {
|
|
const validationMessage = runnableMessage(testCase);
|
|
return (
|
|
<div className="flex min-h-0 flex-1 flex-col items-center justify-center overflow-y-auto px-6 py-8 text-center">
|
|
<div className="w-full max-w-sm">
|
|
<div className="text-lg font-medium tracking-[-0.01em] text-foreground">
|
|
{testCase.name}
|
|
</div>
|
|
<div className="mt-2 flex flex-wrap items-center justify-center gap-2 text-xs text-muted-foreground">
|
|
<span>{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]}</span>
|
|
<span aria-hidden className="text-hairline-strong">·</span>
|
|
<span>{testCase.turns.length} 轮</span>
|
|
{state.evaluatorName && (
|
|
<>
|
|
<span aria-hidden className="text-hairline-strong">·</span>
|
|
<span>{state.evaluatorName}</span>
|
|
</>
|
|
)}
|
|
</div>
|
|
{testCase.description && (
|
|
<p className="mt-3 text-xs leading-5 text-muted-foreground">
|
|
{testCase.description}
|
|
</p>
|
|
)}
|
|
{validationMessage ? (
|
|
<div className="mt-4 rounded-xl border border-amber-500/20 bg-amber-500/[0.06] px-3 py-2 text-xs leading-5 text-amber-600 dark:text-amber-400">
|
|
{validationMessage}
|
|
</div>
|
|
) : null}
|
|
<Button
|
|
type="button"
|
|
className="mx-auto mt-5 h-10 gap-2 rounded-full px-5"
|
|
disabled={Boolean(state.blockedMessage)}
|
|
onClick={onStart}
|
|
title={state.blockedMessage || "开始测试"}
|
|
>
|
|
<Play size={14} />
|
|
开始测试
|
|
</Button>
|
|
{state.blockedMessage && !validationMessage && (
|
|
<p className="mt-2 text-[10px] leading-4 text-muted-soft">
|
|
{state.blockedMessage}
|
|
</p>
|
|
)}
|
|
</div>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
if (
|
|
state.phase === "error" &&
|
|
state.runCase?.executionError &&
|
|
state.runCase.turns.length === 0
|
|
) {
|
|
return (
|
|
<ErrorPanel
|
|
title={state.runCase.executionError.code}
|
|
description={state.runCase.executionError.message}
|
|
/>
|
|
);
|
|
}
|
|
|
|
if (state.phase === "error" && state.error && !state.runCase) {
|
|
return <ErrorPanel title="自动测试启动失败" description={state.error} />;
|
|
}
|
|
|
|
const turns = state.runCase?.turns ?? [];
|
|
const startup = state.runCase?.startup;
|
|
const autoFollowKey = [
|
|
state.phase,
|
|
startup?.completed ? 1 : 0,
|
|
startup?.assistantOutputs.length ?? 0,
|
|
startup?.runtimeRecords?.length ?? 0,
|
|
...turns.map(
|
|
(turn) =>
|
|
`${turn.id}:${turn.runtimeRecords?.length ?? 0}:${turn.toolCalls.length}:${turn.evaluations.length}`,
|
|
),
|
|
state.runCase?.overallCriteria.length ?? 0,
|
|
].join("|");
|
|
const autoFollowResetKey = state.run?.id ?? state.selectedId ?? "auto-test";
|
|
|
|
if (turns.length === 0) {
|
|
if (startup?.completed) {
|
|
return (
|
|
<AutoFollowViewport
|
|
key={autoFollowResetKey}
|
|
followKey={autoFollowKey}
|
|
>
|
|
<AutoTestTimeline
|
|
startup={startup}
|
|
turns={[]}
|
|
maxTurns={state.maxTurns}
|
|
/>
|
|
<div className="flex items-center justify-center gap-2 py-4 text-xs text-muted-foreground">
|
|
<Loader2 size={13} className="animate-spin" />
|
|
正在执行第 1 轮固定输入…
|
|
</div>
|
|
</AutoFollowViewport>
|
|
);
|
|
}
|
|
return (
|
|
<EmptyPanel
|
|
icon={<Loader2 size={20} className="animate-spin" />}
|
|
title={state.stopping ? "正在停止测试…" : "正在启动真实 Pipeline…"}
|
|
description="测试在后端运行，结果会自动同步到这里。"
|
|
/>
|
|
);
|
|
}
|
|
|
|
const evaluating =
|
|
state.phase === "running" && state.currentTurn >= state.maxTurns;
|
|
|
|
return (
|
|
<AutoFollowViewport
|
|
key={autoFollowResetKey}
|
|
followKey={autoFollowKey}
|
|
>
|
|
<div className="flex flex-col gap-4">
|
|
<AutoTestTimeline
|
|
startup={startup}
|
|
turns={turns}
|
|
maxTurns={state.maxTurns}
|
|
/>
|
|
|
|
{state.phase === "running" && (
|
|
<div className="flex items-center justify-center gap-2 py-2 text-xs text-muted-foreground">
|
|
<Loader2 size={13} className="animate-spin" />
|
|
{state.stopping
|
|
? "正在停止测试…"
|
|
: evaluating
|
|
? "对话已完成，正在评估结果…"
|
|
: `正在执行第 ${state.currentTurn + 1} 轮…`}
|
|
</div>
|
|
)}
|
|
|
|
{state.phase === "error" && state.runCase?.executionError && (
|
|
<div className="rounded-xl border border-amber-500/25 bg-amber-500/[0.06] px-3 py-2.5 text-xs leading-5 text-amber-600 dark:text-amber-400">
|
|
<span className="font-medium">
|
|
{state.runCase.executionError.code}
|
|
</span>
|
|
<span> · {state.runCase.executionError.message}</span>
|
|
</div>
|
|
)}
|
|
|
|
{state.runCase && state.runCase.overallCriteria.length > 0 && (
|
|
<div className="space-y-2 border-t border-hairline pt-3">
|
|
<div className="text-[11px] font-medium text-muted-soft">
|
|
整体评估标准
|
|
</div>
|
|
{state.runCase.overallCriteria.map((evaluation) => (
|
|
<EvaluationDetails
|
|
key={evaluation.id}
|
|
evaluation={evaluation}
|
|
/>
|
|
))}
|
|
</div>
|
|
)}
|
|
</div>
|
|
</AutoFollowViewport>
|
|
);
|
|
}
|
|
|
|
function startupAssistantMessages(startup: BatchStartupResult): string[] {
|
|
const greeting = startup.greeting.trim();
|
|
const greetingText = normalizeDisplayText(greeting);
|
|
const seen = new Set<string>();
|
|
const assistantMessages = startup.assistantOutputs.filter((output) => {
|
|
const normalizedOutput = normalizeDisplayText(output);
|
|
if (
|
|
!normalizedOutput ||
|
|
normalizedOutput === greetingText ||
|
|
seen.has(normalizedOutput)
|
|
) {
|
|
return false;
|
|
}
|
|
seen.add(normalizedOutput);
|
|
return true;
|
|
});
|
|
return [greeting, ...assistantMessages].filter(Boolean);
|
|
}
|
|
|
|
function runtimeRecordKey(
|
|
event: Record<string, unknown>,
|
|
prefix: string,
|
|
index: number,
|
|
): string {
|
|
const id = typeof event.eventId === "string" ? event.eventId : "";
|
|
const name = typeof event.event === "string" ? event.event : "record";
|
|
return id || `${prefix}-${name}-${index}`;
|
|
}
|
|
|
|
function normalizeDisplayText(value: string): string {
|
|
return value.trim().replace(/\s+/g, " ");
|
|
}
|
|
|
|
function AutoTestTimeline({
|
|
startup,
|
|
turns,
|
|
maxTurns,
|
|
}: {
|
|
startup?: BatchStartupResult;
|
|
turns: BatchRunCase["turns"];
|
|
maxTurns: number;
|
|
}) {
|
|
const items: DebugTimelineItem[] = [];
|
|
|
|
if (startup?.completed) {
|
|
items.push({
|
|
id: "startup-boundary",
|
|
kind: "content",
|
|
content: (
|
|
<div className="flex items-center gap-2 py-1 text-[11px] text-muted-soft">
|
|
<span className="h-px flex-1 bg-hairline" />
|
|
<span>开场阶段 · 自动执行，不计入脚本断言</span>
|
|
<span className="h-px flex-1 bg-hairline" />
|
|
</div>
|
|
),
|
|
});
|
|
|
|
const openingMessages = startupAssistantMessages(startup);
|
|
const records = startup.runtimeRecords ?? [];
|
|
const firstMessageStartedIndex = records.findIndex(
|
|
(event) => event.event === "message_started",
|
|
);
|
|
const firstMessageCompletedIndex = records.findIndex(
|
|
(event) => event.event === "message_completed",
|
|
);
|
|
const appendOpeningMessage = (message: string, index: number) => {
|
|
items.push({
|
|
id: `startup-message-${index}`,
|
|
kind: "message",
|
|
message: {
|
|
id: `startup-message-${index}`,
|
|
role: "assistant",
|
|
content: message,
|
|
},
|
|
});
|
|
};
|
|
|
|
const appendRuntimeRecord = (
|
|
event: Record<string, unknown>,
|
|
index: number,
|
|
) => {
|
|
const id = runtimeRecordKey(event, "startup-record", index);
|
|
items.push({
|
|
id,
|
|
kind: "runtime-record",
|
|
record: {
|
|
id,
|
|
event,
|
|
timestamp:
|
|
typeof event.timestamp === "string" ? event.timestamp : undefined,
|
|
},
|
|
});
|
|
};
|
|
|
|
if (firstMessageStartedIndex < 0) {
|
|
records.forEach(appendRuntimeRecord);
|
|
openingMessages.forEach(appendOpeningMessage);
|
|
} else {
|
|
records.forEach((event, index) => {
|
|
appendRuntimeRecord(event, index);
|
|
if (index === firstMessageStartedIndex && openingMessages[0]) {
|
|
appendOpeningMessage(openingMessages[0], 0);
|
|
}
|
|
if (index === firstMessageCompletedIndex) {
|
|
openingMessages.slice(1).forEach((message, outputIndex) => {
|
|
appendOpeningMessage(message, outputIndex + 1);
|
|
});
|
|
}
|
|
});
|
|
if (firstMessageCompletedIndex < 0) {
|
|
openingMessages.slice(1).forEach((message, outputIndex) => {
|
|
appendOpeningMessage(message, outputIndex + 1);
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
turns.forEach((turn) => {
|
|
items.push({
|
|
id: `turn-user-${turn.id}`,
|
|
kind: "message",
|
|
message: {
|
|
id: `turn-user-${turn.id}`,
|
|
role: "user",
|
|
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
|
|
content: turn.userInput,
|
|
attachments: turn.image
|
|
? [
|
|
{
|
|
id: turn.image.assetId,
|
|
type: "image",
|
|
url: testAssetsApi.contentUrl(turn.image.assetId),
|
|
alt: turn.image.filename,
|
|
},
|
|
]
|
|
: undefined,
|
|
},
|
|
});
|
|
items.push({
|
|
id: `turn-assistant-${turn.id}`,
|
|
kind: "message",
|
|
message: {
|
|
id: `turn-assistant-${turn.id}`,
|
|
role: "assistant",
|
|
content: turn.assistantReply || "（没有生成文字回复）",
|
|
},
|
|
});
|
|
turn.runtimeRecords?.forEach((event, index) => {
|
|
const id = runtimeRecordKey(event, `turn-record-${turn.id}`, index);
|
|
items.push({
|
|
id,
|
|
kind: "runtime-record",
|
|
record: {
|
|
id,
|
|
event,
|
|
timestamp:
|
|
typeof event.timestamp === "string" ? event.timestamp : undefined,
|
|
},
|
|
});
|
|
});
|
|
if (turn.toolCalls.length > 0) {
|
|
items.push({
|
|
id: `turn-tools-${turn.id}`,
|
|
kind: "content",
|
|
content: <AutoTestToolDetails toolCalls={turn.toolCalls} />,
|
|
});
|
|
}
|
|
if (turn.evaluations.length > 0) {
|
|
items.push({
|
|
id: `turn-evaluations-${turn.id}`,
|
|
kind: "content",
|
|
content: (
|
|
<div className="w-full space-y-2">
|
|
{turn.evaluations.map((evaluation) => (
|
|
<EvaluationDetails
|
|
key={evaluation.id}
|
|
evaluation={evaluation}
|
|
/>
|
|
))}
|
|
</div>
|
|
),
|
|
});
|
|
}
|
|
});
|
|
|
|
return <DebugTimeline items={items} recordContext="auto-test" />;
|
|
}
|
|
|
|
function AutoTestToolDetails({
|
|
toolCalls,
|
|
}: {
|
|
toolCalls: BatchRunCase["turns"][number]["toolCalls"];
|
|
}) {
|
|
return (
|
|
<details className="w-full rounded-xl border border-hairline bg-canvas-soft px-3 py-2.5 text-xs">
|
|
<summary className="flex cursor-pointer list-none items-center gap-1.5 font-medium text-foreground">
|
|
<Wrench size={12} />
|
|
工具调用详情
|
|
<span className="ml-auto text-[10px] font-normal text-muted-soft">
|
|
{toolCalls.length} 次
|
|
</span>
|
|
</summary>
|
|
<div className="mt-2 space-y-3 border-t border-hairline pt-2">
|
|
{toolCalls.map((toolCall) => (
|
|
<div key={toolCall.id} className="text-[11px] leading-5">
|
|
<div>
|
|
<code className="text-foreground">{toolCall.functionName}</code>
|
|
<span className="ml-2 text-muted-soft">
|
|
{toolCall.outcome === "success" ? "成功" : "错误"} ·{" "}
|
|
{toolCall.durationMs} ms
|
|
</span>
|
|
</div>
|
|
<div className="mt-1 grid gap-1.5">
|
|
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
|
|
{toolCall.argumentsJson}
|
|
</pre>
|
|
<pre className="max-h-28 overflow-auto whitespace-pre-wrap break-all rounded-lg bg-background/80 p-2 font-mono text-[10px] text-muted-foreground">
|
|
{toolCall.resultJson}
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
))}
|
|
</div>
|
|
</details>
|
|
);
|
|
}
|
|
|
|
function EvaluationDetails({
|
|
evaluation,
|
|
}: {
|
|
evaluation: BatchEvaluationResult;
|
|
}) {
|
|
const passed = evaluation.status === "pass";
|
|
return (
|
|
<div
|
|
className={cn(
|
|
"rounded-xl border bg-canvas-soft px-3 py-2.5",
|
|
passed
|
|
? "border-hairline"
|
|
: "border-destructive/25 bg-destructive/5",
|
|
)}
|
|
>
|
|
<div className="flex items-center gap-2">
|
|
<span
|
|
className={cn(
|
|
"flex h-4 w-4 shrink-0 items-center justify-center rounded-full",
|
|
passed
|
|
? "bg-success/10 text-success"
|
|
: "bg-destructive/10 text-destructive",
|
|
)}
|
|
>
|
|
{passed ? <Check size={10} /> : <X size={10} />}
|
|
</span>
|
|
<div className="min-w-0 flex-1 text-[11px] font-medium text-foreground">
|
|
{evaluation.label}
|
|
</div>
|
|
<span
|
|
className={cn(
|
|
"text-[10px] font-medium",
|
|
passed ? "text-success" : "text-destructive",
|
|
)}
|
|
>
|
|
{passed ? "通过" : "未通过"}
|
|
</span>
|
|
</div>
|
|
<dl className="mt-2 grid grid-cols-[52px_minmax(0,1fr)] gap-x-2 gap-y-1 text-[11px] leading-5">
|
|
<dt className="text-muted-soft">评估标准</dt>
|
|
<dd className="whitespace-pre-wrap break-words text-muted-foreground">
|
|
{evaluation.expected || "（未提供评估标准）"}
|
|
</dd>
|
|
<dt className="text-muted-soft">实际结果</dt>
|
|
<dd className="whitespace-pre-wrap break-words text-muted-foreground">
|
|
{evaluation.actual || "（没有可评估内容）"}
|
|
</dd>
|
|
{evaluation.reason && (
|
|
<>
|
|
<dt className={passed ? "text-muted-soft" : "text-destructive"}>
|
|
判断说明
|
|
</dt>
|
|
<dd className="whitespace-pre-wrap break-words text-foreground">
|
|
{evaluation.reason}
|
|
</dd>
|
|
</>
|
|
)}
|
|
</dl>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
function EmptyPanel({
|
|
icon,
|
|
title,
|
|
description,
|
|
}: {
|
|
icon: React.ReactNode;
|
|
title: string;
|
|
description: string;
|
|
}) {
|
|
return (
|
|
<div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-8 text-center">
|
|
<span className="text-muted-soft">{icon}</span>
|
|
<div className="mt-2 text-sm font-medium text-foreground">{title}</div>
|
|
<p className="mx-auto mt-1 max-w-xs text-xs leading-5 text-muted-foreground">
|
|
{description}
|
|
</p>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
function ErrorPanel({ title, description }: { title: string; description: string }) {
|
|
return (
|
|
<div className="flex min-h-0 flex-1 flex-col items-center justify-center px-6 py-8 text-center">
|
|
<span className="flex h-9 w-9 items-center justify-center rounded-full bg-amber-500/10 text-amber-600 dark:text-amber-400">
|
|
<TriangleAlert size={18} />
|
|
</span>
|
|
<div className="mt-3 text-sm font-medium text-foreground">{title}</div>
|
|
<p className="mx-auto mt-1 max-w-sm text-xs leading-5 text-muted-foreground">
|
|
{description}
|
|
</p>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
// ─── Bottom run bar ─────────────────────────────────────────────────────────
|
|
|
|
export function AutoTestRunBar({
|
|
state,
|
|
testCase,
|
|
suites,
|
|
cases,
|
|
loading,
|
|
onSelectCase,
|
|
onClearCase,
|
|
onStop,
|
|
onRerun,
|
|
}: {
|
|
state: AutoTestRunState;
|
|
testCase: TestCase | null;
|
|
suites: TestSuite[];
|
|
cases: TestCase[];
|
|
loading: boolean;
|
|
onSelectCase: (id: string) => void;
|
|
onClearCase: () => void;
|
|
onStop: () => void;
|
|
onRerun: () => void;
|
|
}) {
|
|
const pickerTrigger = (
|
|
<button
|
|
type="button"
|
|
className="flex min-w-0 flex-1 items-center gap-2 rounded-full px-2 py-1.5 text-left transition-colors hover:bg-canvas-soft"
|
|
>
|
|
{loading ? (
|
|
<Loader2 size={15} className="shrink-0 animate-spin text-muted-soft" />
|
|
) : (
|
|
<FlaskConical size={16} className="shrink-0 text-muted-soft" />
|
|
)}
|
|
<span
|
|
className={cn(
|
|
"min-w-0 flex-1 truncate text-sm",
|
|
testCase ? "font-medium text-foreground" : "text-muted-foreground",
|
|
)}
|
|
>
|
|
{loading ? "加载测试用例…" : testCase?.name ?? "选择测试用例…"}
|
|
</span>
|
|
<ChevronDown size={14} className="shrink-0 text-muted-soft" />
|
|
</button>
|
|
);
|
|
|
|
if (state.phase === "starting" || state.phase === "running") {
|
|
return (
|
|
<div className="flex min-h-11 items-center gap-2 rounded-[1.4rem] border border-hairline-strong bg-background px-3">
|
|
<Loader2 size={15} className="shrink-0 animate-spin text-primary" />
|
|
<span className="min-w-0 flex-1 truncate text-sm font-medium text-foreground">
|
|
{testCase?.name}
|
|
</span>
|
|
<span className="shrink-0 text-xs tabular-nums text-muted-foreground">
|
|
{state.stopping
|
|
? "停止中"
|
|
: state.phase === "starting"
|
|
? "启动中"
|
|
: `${state.currentTurn} / ${state.maxTurns}`}
|
|
</span>
|
|
<Button
|
|
type="button"
|
|
size="sm"
|
|
variant="secondary"
|
|
className="h-8 shrink-0 gap-1.5 rounded-full px-3"
|
|
disabled={state.stopping}
|
|
onClick={onStop}
|
|
>
|
|
<Square size={11} className="fill-current" />
|
|
停止
|
|
</Button>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
if (
|
|
state.phase === "passed" ||
|
|
state.phase === "failed" ||
|
|
state.phase === "error" ||
|
|
state.phase === "cancelled"
|
|
) {
|
|
const passed = state.phase === "passed";
|
|
const label = passed
|
|
? `PASS · ${state.currentTurn} 轮`
|
|
: state.phase === "failed"
|
|
? "FAIL"
|
|
: state.phase === "cancelled"
|
|
? "已停止"
|
|
: "执行错误";
|
|
return (
|
|
<div className="flex min-h-11 items-center gap-2 rounded-[1.4rem] border border-hairline-strong bg-background px-3">
|
|
<span
|
|
className={cn(
|
|
"flex h-5 w-5 shrink-0 items-center justify-center rounded-full",
|
|
passed
|
|
? "bg-success/15 text-success"
|
|
: state.phase === "failed"
|
|
? "bg-destructive/15 text-destructive"
|
|
: "bg-amber-500/10 text-amber-600 dark:text-amber-400",
|
|
)}
|
|
>
|
|
{passed ? <Check size={12} /> : <X size={12} />}
|
|
</span>
|
|
<span className="min-w-0 flex-1 truncate text-sm font-medium text-foreground">
|
|
{testCase?.name}
|
|
</span>
|
|
<span
|
|
className={cn(
|
|
"shrink-0 text-xs font-medium",
|
|
passed
|
|
? "text-success"
|
|
: state.phase === "failed"
|
|
? "text-destructive"
|
|
: "text-amber-600 dark:text-amber-400",
|
|
)}
|
|
>
|
|
{label}
|
|
</span>
|
|
<Button
|
|
type="button"
|
|
size="sm"
|
|
variant="secondary"
|
|
className="h-8 shrink-0 rounded-full px-3 text-xs"
|
|
disabled={Boolean(state.blockedMessage)}
|
|
onClick={onRerun}
|
|
title={state.blockedMessage || "再次运行"}
|
|
>
|
|
再次运行
|
|
</Button>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
return (
|
|
<div className="space-y-1.5">
|
|
<div className="flex h-11 items-center gap-1.5 rounded-[1.4rem] border border-hairline-strong bg-background px-2">
|
|
<div className="flex min-w-0 flex-1 items-center">
|
|
<CasePicker
|
|
suites={suites}
|
|
cases={cases}
|
|
selectedId={state.selectedId}
|
|
loading={loading}
|
|
onSelect={onSelectCase}
|
|
trigger={pickerTrigger}
|
|
/>
|
|
{testCase && (
|
|
<button
|
|
type="button"
|
|
onClick={onClearCase}
|
|
aria-label="清除当前用例"
|
|
className="mr-1 flex h-7 w-7 shrink-0 items-center justify-center rounded-full text-muted-soft transition-colors hover:bg-surface-strong hover:text-foreground"
|
|
>
|
|
<X size={14} />
|
|
</button>
|
|
)}
|
|
</div>
|
|
</div>
|
|
{state.blockedMessage && testCase && (
|
|
<p className="truncate px-3 text-[10px] leading-4 text-muted-soft">
|
|
{state.blockedMessage}
|
|
</p>
|
|
)}
|
|
</div>
|
|
);
|
|
}
|
|
|
|
// ─── Hook: real single-case run ─────────────────────────────────────────────
|
|
|
|
export function useAutoTestRunner({
|
|
assistantId,
|
|
enabled,
|
|
hasUnsavedChanges,
|
|
dynamicVariables,
|
|
dynamicVariablesError,
|
|
}: AutoTestRunnerOptions) {
|
|
const [suites, setSuites] = useState<TestSuite[]>([]);
|
|
const [cases, setCases] = useState<TestCase[]>([]);
|
|
const [evaluatorModels, setEvaluatorModels] = useState<ModelResource[]>([]);
|
|
const [evaluatorModelId, setEvaluatorModelId] = useState("");
|
|
const [catalogLoading, setCatalogLoading] = useState(true);
|
|
const [catalogError, setCatalogError] = useState("");
|
|
const [selectedId, setSelectedId] = useState<string | null>(null);
|
|
const [run, setRun] = useState<BatchRunSnapshot | null>(null);
|
|
const activeRunRef = useRef<BatchRunSnapshot | null>(null);
|
|
const [starting, setStarting] = useState(false);
|
|
const [stopping, setStopping] = useState(false);
|
|
const [runError, setRunError] = useState("");
|
|
|
|
useEffect(() => {
|
|
activeRunRef.current = run;
|
|
}, [run]);
|
|
|
|
useEffect(
|
|
() => () => {
|
|
const activeRun = activeRunRef.current;
|
|
if (activeRun && !isRunFinished(activeRun)) {
|
|
void batchRunsApi.cancel(activeRun.id);
|
|
}
|
|
},
|
|
[],
|
|
);
|
|
|
|
useEffect(() => {
|
|
if (!enabled) return;
|
|
let disposed = false;
|
|
|
|
void Promise.all([
|
|
testSuitesApi.list(),
|
|
testCasesApi.list(),
|
|
modelResourcesApi.list(),
|
|
])
|
|
.then(([loadedSuites, loadedCases, resources]) => {
|
|
if (disposed) return;
|
|
const llms = resources.filter(
|
|
(item) => item.capability === "LLM" && item.enabled,
|
|
);
|
|
setSuites(loadedSuites);
|
|
setCases(loadedCases);
|
|
setEvaluatorModels(llms);
|
|
setCatalogError("");
|
|
const evaluator = llms.find((item) => item.isDefault) ?? llms[0];
|
|
setEvaluatorModelId(evaluator?.id ?? "");
|
|
})
|
|
.catch((error) => {
|
|
if (disposed) return;
|
|
setCatalogError(
|
|
error instanceof Error ? error.message : "加载测试用例失败",
|
|
);
|
|
})
|
|
.finally(() => {
|
|
if (!disposed) setCatalogLoading(false);
|
|
});
|
|
|
|
return () => {
|
|
disposed = true;
|
|
};
|
|
}, [enabled]);
|
|
|
|
useEffect(() => {
|
|
if (!enabled || !run || isRunFinished(run)) return;
|
|
let disposed = false;
|
|
let timer = 0;
|
|
|
|
async function poll() {
|
|
try {
|
|
const snapshot = await batchRunsApi.get(run!.id);
|
|
if (disposed) return;
|
|
setRun(snapshot);
|
|
setRunError("");
|
|
if (isRunFinished(snapshot)) {
|
|
setStopping(false);
|
|
return;
|
|
}
|
|
} catch (error) {
|
|
if (!disposed) {
|
|
setRunError(
|
|
error instanceof Error ? error.message : "刷新测试结果失败",
|
|
);
|
|
}
|
|
}
|
|
if (!disposed) timer = window.setTimeout(() => void poll(), RUN_POLL_MS);
|
|
}
|
|
|
|
timer = window.setTimeout(() => void poll(), RUN_POLL_MS);
|
|
return () => {
|
|
disposed = true;
|
|
window.clearTimeout(timer);
|
|
};
|
|
}, [enabled, run]);
|
|
|
|
const selectedCase = useMemo(
|
|
() => cases.find((item) => item.id === selectedId) ?? null,
|
|
[cases, selectedId],
|
|
);
|
|
const evaluator = useMemo(
|
|
() => evaluatorModels.find((item) => item.id === evaluatorModelId) ?? null,
|
|
[evaluatorModelId, evaluatorModels],
|
|
);
|
|
const runCase = run?.cases[0] ?? null;
|
|
|
|
let blockedMessage = "";
|
|
if (catalogLoading) blockedMessage = "正在加载测试用例…";
|
|
else if (catalogError) blockedMessage = catalogError;
|
|
else if (!assistantId) blockedMessage = "请先保存助手，再开始自动测试";
|
|
else if (hasUnsavedChanges) blockedMessage = "请先保存当前改动，再开始自动测试";
|
|
else if (dynamicVariablesError) blockedMessage = dynamicVariablesError;
|
|
else if (!selectedCase) blockedMessage = "请先选择测试用例";
|
|
else if (runnableMessage(selectedCase)) {
|
|
blockedMessage = runnableMessage(selectedCase);
|
|
} else if (!evaluatorModelId) blockedMessage = "请先配置可用的 LLM 评估模型";
|
|
|
|
let phase: AutoTestPhase = selectedCase ? "ready" : "idle";
|
|
if (starting) phase = "starting";
|
|
else if (run) {
|
|
if (run.status === "cancelled" || runCase?.status === "skipped") {
|
|
phase = "cancelled";
|
|
} else if (runCase?.status === "pass") phase = "passed";
|
|
else if (runCase?.status === "fail") phase = "failed";
|
|
else if (runCase?.status === "error") phase = "error";
|
|
else phase = "running";
|
|
} else if (runError) phase = "error";
|
|
|
|
const state: AutoTestRunState = {
|
|
phase,
|
|
selectedId,
|
|
currentTurn: runCase?.turns.length ?? 0,
|
|
maxTurns: selectedCase?.turns.length ?? 0,
|
|
run,
|
|
runCase,
|
|
error: runError,
|
|
stopping,
|
|
blockedMessage,
|
|
evaluatorName: evaluator?.name ?? "",
|
|
};
|
|
|
|
const cancelActiveRun = useCallback(async () => {
|
|
if (!run || isRunFinished(run)) return;
|
|
setStopping(true);
|
|
try {
|
|
const snapshot = await batchRunsApi.cancel(run.id);
|
|
setRun(snapshot);
|
|
setRunError("");
|
|
if (isRunFinished(snapshot)) setStopping(false);
|
|
} catch (error) {
|
|
setRunError(error instanceof Error ? error.message : "停止测试失败");
|
|
setStopping(false);
|
|
}
|
|
}, [run]);
|
|
|
|
const selectCase = useCallback(
|
|
(id: string) => {
|
|
if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id);
|
|
setSelectedId(id);
|
|
setRun(null);
|
|
setRunError("");
|
|
setStarting(false);
|
|
setStopping(false);
|
|
},
|
|
[run],
|
|
);
|
|
|
|
const clearCase = useCallback(() => {
|
|
if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id);
|
|
setSelectedId(null);
|
|
setRun(null);
|
|
setRunError("");
|
|
setStarting(false);
|
|
setStopping(false);
|
|
}, [run]);
|
|
|
|
const start = useCallback(async () => {
|
|
if (
|
|
!assistantId ||
|
|
!selectedCase ||
|
|
!evaluatorModelId ||
|
|
blockedMessage ||
|
|
starting
|
|
) {
|
|
return;
|
|
}
|
|
setStarting(true);
|
|
setRun(null);
|
|
setRunError("");
|
|
try {
|
|
const snapshot = await batchRunsApi.create({
|
|
assistantId,
|
|
evaluatorModelResourceId: evaluatorModelId,
|
|
caseIds: [selectedCase.id],
|
|
dynamicVariables,
|
|
title: `调试 · ${selectedCase.name}`,
|
|
config: {
|
|
concurrency: 1,
|
|
timeoutSecs: 60,
|
|
failureStrategy: "continue",
|
|
errorRetryCount: 0,
|
|
errorStrategy: "continue",
|
|
},
|
|
});
|
|
setRun(snapshot);
|
|
} catch (error) {
|
|
setRunError(error instanceof Error ? error.message : "启动自动测试失败");
|
|
} finally {
|
|
setStarting(false);
|
|
}
|
|
}, [
|
|
assistantId,
|
|
blockedMessage,
|
|
dynamicVariables,
|
|
evaluatorModelId,
|
|
selectedCase,
|
|
starting,
|
|
]);
|
|
|
|
const stop = useCallback(() => {
|
|
if (run && !isRunFinished(run)) {
|
|
void cancelActiveRun();
|
|
return;
|
|
}
|
|
setRun(null);
|
|
setRunError("");
|
|
}, [cancelActiveRun, run]);
|
|
|
|
const rerun = useCallback(() => {
|
|
setRun(null);
|
|
setRunError("");
|
|
window.setTimeout(() => void start(), 0);
|
|
}, [start]);
|
|
|
|
return {
|
|
state,
|
|
suites,
|
|
cases,
|
|
catalogLoading,
|
|
selectedCase,
|
|
selectCase,
|
|
clearCase,
|
|
start: () => void start(),
|
|
stop,
|
|
rerun,
|
|
};
|
|
}
|