"use client"; /** * Debug Drawer 的真实自动测试客户端。 * * 它复用批量测试的持久化 Test Run API,但每次只运行一个用例。这样 Drawer * 与批量测试共享同一套 Pipeline、工具 Mock、评估器和运行记录,不维护第二套 * 测试语义。 */ import type React from "react"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { ArrowDown, Check, ChevronDown, FlaskConical, Loader2, Play, Square, TriangleAlert, Wrench, X, } from "lucide-react"; import { DebugTimeline, type DebugTimelineItem, } from "@/components/debug-timeline"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { Popover, PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; import { SearchInput } from "@/components/ui/search-input"; import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; import type { BatchEvaluationResult, BatchRunCase, BatchRunSnapshot, BatchStartupResult, } from "@/data/batch-run"; import { getTestCaseValidationMessage, TEST_CASE_INPUT_MODE_SHORT_LABEL, type TestCase, type TestSuite, } from "@/data/test-suites"; import { batchRunsApi, modelResourcesApi, testAssetsApi, testCasesApi, testSuitesApi, type ModelResource, } from "@/lib/api"; import { cn } from "@/lib/utils"; const RUN_POLL_MS = 1000; const AUTO_FOLLOW_BOTTOM_THRESHOLD_PX = 32; export type AutoTestPhase = | "idle" | "ready" | "starting" | "running" | "passed" | "failed" | "error" | "cancelled"; export type AutoTestRunState = { phase: AutoTestPhase; selectedId: string | null; currentTurn: number; maxTurns: number; run: BatchRunSnapshot | null; runCase: BatchRunCase | null; error: string; stopping: boolean; blockedMessage: string; evaluatorName: string; }; type AutoTestRunnerOptions = { assistantId: string | null; enabled: boolean; hasUnsavedChanges: boolean; dynamicVariables: Record; dynamicVariablesError: string; }; function isRunFinished(run: BatchRunSnapshot): boolean { return ( Boolean(run.finishedAt) || run.status === "completed" || run.status === "cancelled" ); } function runnableMessage(item: TestCase): string { if (item.inputMode !== "fixed_script_text") { return "当前自动测试仅支持固定脚本 · 多模态"; } return getTestCaseValidationMessage(item) ?? ""; } // ─── Mode tabs ────────────────────────────────────────────────────────────── export function DebugModeTabs({ mode, onChange, }: { mode: "manual" | "auto"; onChange: (mode: "manual" | "auto") => void; }) { return ( onChange(value as "manual" | "auto")} className="w-full gap-0" > 手动测试 自动测试 ); } // ─── Case picker ──────────────────────────────────────────────────────────── function CasePicker({ suites, cases, selectedId, loading, onSelect, trigger, }: { suites: TestSuite[]; cases: TestCase[]; selectedId: string | null; loading: boolean; onSelect: (id: string) => void; trigger: React.ReactNode; }) { const [open, setOpen] = useState(false); const [query, setQuery] = useState(""); const normalizedQuery = query.trim().toLocaleLowerCase(); const visibleSuites = useMemo( () => suites .map((suite) => ({ suite, cases: cases.filter((item) => item.suiteId === suite.id), })) .map((group) => ({ ...group, cases: group.cases.filter((item) => { if (!normalizedQuery) return true; return [ item.name, item.description, TEST_CASE_INPUT_MODE_SHORT_LABEL[item.inputMode], ].some((value) => value.toLocaleLowerCase().includes(normalizedQuery)); }), })) .filter((group) => group.cases.length > 0), [cases, normalizedQuery, suites], ); return ( { setOpen(next); if (!next) setQuery(""); }} > {trigger}
选择测试用例
{loading ? (
加载真实测试用例…
) : visibleSuites.length === 0 ? (
{cases.length === 0 ? "还没有测试用例" : "没有匹配的测试用例"}
) : ( visibleSuites.map(({ suite, cases: suiteCases }) => (
{suite.name} {suiteCases.length}
{suiteCases.map((item) => { const unsupported = Boolean(runnableMessage(item)); return ( ); })}
)) )}
); } // ─── Result panel ─────────────────────────────────────────────────────────── function AutoFollowViewport({ children, followKey, }: { children: React.ReactNode; followKey: string; }) { const scrollRef = useRef(null); const contentRef = useRef(null); const isFollowingRef = useRef(true); const [isFollowing, setIsFollowing] = useState(true); const updateFollowing = useCallback((next: boolean) => { isFollowingRef.current = next; setIsFollowing((current) => (current === next ? current : next)); }, []); const scrollToLatest = useCallback(() => { const viewport = scrollRef.current; if (!viewport) return; viewport.scrollTop = viewport.scrollHeight; }, []); const handleScroll = useCallback(() => { const viewport = scrollRef.current; if (!viewport) return; const distanceFromBottom = viewport.scrollHeight - viewport.scrollTop - viewport.clientHeight; updateFollowing(distanceFromBottom <= AUTO_FOLLOW_BOTTOM_THRESHOLD_PX); }, [updateFollowing]); useEffect(() => { if (!isFollowingRef.current) return; const frame = window.requestAnimationFrame(scrollToLatest); return () => window.cancelAnimationFrame(frame); }, [followKey, scrollToLatest]); // Runtime cards and images can grow after the test snapshot is rendered. useEffect(() => { const content = contentRef.current; if (!content) return; const observer = new ResizeObserver(() => { if (isFollowingRef.current) scrollToLatest(); }); observer.observe(content); return () => observer.disconnect(); }, [scrollToLatest]); const resumeFollowing = useCallback(() => { updateFollowing(true); scrollToLatest(); }, [scrollToLatest, updateFollowing]); return (
{children}
{!isFollowing && ( )}
); } export function AutoTestLivePanel({ testCase, state, onStart, }: { testCase: TestCase | null; state: AutoTestRunState; onStart: () => void; }) { if (!testCase) { return (
选择测试用例后开始
); } if (state.phase === "idle" || state.phase === "ready") { const validationMessage = runnableMessage(testCase); return (
{testCase.name}
{TEST_CASE_INPUT_MODE_SHORT_LABEL[testCase.inputMode]} · {testCase.turns.length} 轮 {state.evaluatorName && ( <> · {state.evaluatorName} )}
{testCase.description && (

{testCase.description}

)} {validationMessage ? (
{validationMessage}
) : null} {state.blockedMessage && !validationMessage && (

{state.blockedMessage}

)}
); } if ( state.phase === "error" && state.runCase?.executionError && state.runCase.turns.length === 0 ) { return ( ); } if (state.phase === "error" && state.error && !state.runCase) { return ; } const turns = state.runCase?.turns ?? []; const startup = state.runCase?.startup; const autoFollowKey = [ state.phase, startup?.completed ? 1 : 0, startup?.assistantOutputs.length ?? 0, startup?.runtimeRecords?.length ?? 0, ...turns.map( (turn) => `${turn.id}:${turn.runtimeRecords?.length ?? 0}:${turn.toolCalls.length}:${turn.evaluations.length}`, ), state.runCase?.overallCriteria.length ?? 0, ].join("|"); const autoFollowResetKey = state.run?.id ?? state.selectedId ?? "auto-test"; if (turns.length === 0) { if (startup?.completed) { return (
正在执行第 1 轮固定输入…
); } return ( } title={state.stopping ? "正在停止测试…" : "正在启动真实 Pipeline…"} description="测试在后端运行,结果会自动同步到这里。" /> ); } const evaluating = state.phase === "running" && state.currentTurn >= state.maxTurns; return (
{state.phase === "running" && (
{state.stopping ? "正在停止测试…" : evaluating ? "对话已完成,正在评估结果…" : `正在执行第 ${state.currentTurn + 1} 轮…`}
)} {state.phase === "error" && state.runCase?.executionError && (
{state.runCase.executionError.code} · {state.runCase.executionError.message}
)} {state.runCase && state.runCase.overallCriteria.length > 0 && (
整体评估标准
{state.runCase.overallCriteria.map((evaluation) => ( ))}
)}
); } function startupAssistantMessages(startup: BatchStartupResult): string[] { const greeting = startup.greeting.trim(); const greetingText = normalizeDisplayText(greeting); const seen = new Set(); const assistantMessages = startup.assistantOutputs.filter((output) => { const normalizedOutput = normalizeDisplayText(output); if ( !normalizedOutput || normalizedOutput === greetingText || seen.has(normalizedOutput) ) { return false; } seen.add(normalizedOutput); return true; }); return [greeting, ...assistantMessages].filter(Boolean); } function runtimeRecordKey( event: Record, prefix: string, index: number, ): string { const id = typeof event.eventId === "string" ? event.eventId : ""; const name = typeof event.event === "string" ? event.event : "record"; return id || `${prefix}-${name}-${index}`; } function normalizeDisplayText(value: string): string { return value.trim().replace(/\s+/g, " "); } function AutoTestTimeline({ startup, turns, maxTurns, }: { startup?: BatchStartupResult; turns: BatchRunCase["turns"]; maxTurns: number; }) { const items: DebugTimelineItem[] = []; if (startup?.completed) { items.push({ id: "startup-boundary", kind: "content", content: (
开场阶段 · 自动执行,不计入脚本断言
), }); const openingMessages = startupAssistantMessages(startup); const records = startup.runtimeRecords ?? []; const firstMessageStartedIndex = records.findIndex( (event) => event.event === "message_started", ); const firstMessageCompletedIndex = records.findIndex( (event) => event.event === "message_completed", ); const appendOpeningMessage = (message: string, index: number) => { items.push({ id: `startup-message-${index}`, kind: "message", message: { id: `startup-message-${index}`, role: "assistant", content: message, }, }); }; const appendRuntimeRecord = ( event: Record, index: number, ) => { const id = runtimeRecordKey(event, "startup-record", index); items.push({ id, kind: "runtime-record", record: { id, event, timestamp: typeof event.timestamp === "string" ? event.timestamp : undefined, }, }); }; if (firstMessageStartedIndex < 0) { records.forEach(appendRuntimeRecord); openingMessages.forEach(appendOpeningMessage); } else { records.forEach((event, index) => { appendRuntimeRecord(event, index); if (index === firstMessageStartedIndex && openingMessages[0]) { appendOpeningMessage(openingMessages[0], 0); } if (index === firstMessageCompletedIndex) { openingMessages.slice(1).forEach((message, outputIndex) => { appendOpeningMessage(message, outputIndex + 1); }); } }); if (firstMessageCompletedIndex < 0) { openingMessages.slice(1).forEach((message, outputIndex) => { appendOpeningMessage(message, outputIndex + 1); }); } } } turns.forEach((turn) => { items.push({ id: `turn-user-${turn.id}`, kind: "message", message: { id: `turn-user-${turn.id}`, role: "user", label: `固定输入 · ${turn.index + 1} / ${maxTurns}`, content: turn.userInput, attachments: turn.image ? [ { id: turn.image.assetId, type: "image", url: testAssetsApi.contentUrl(turn.image.assetId), alt: turn.image.filename, }, ] : undefined, }, }); items.push({ id: `turn-assistant-${turn.id}`, kind: "message", message: { id: `turn-assistant-${turn.id}`, role: "assistant", content: turn.assistantReply || "(没有生成文字回复)", }, }); turn.runtimeRecords?.forEach((event, index) => { const id = runtimeRecordKey(event, `turn-record-${turn.id}`, index); items.push({ id, kind: "runtime-record", record: { id, event, timestamp: typeof event.timestamp === "string" ? event.timestamp : undefined, }, }); }); if (turn.toolCalls.length > 0) { items.push({ id: `turn-tools-${turn.id}`, kind: "content", content: , }); } if (turn.evaluations.length > 0) { items.push({ id: `turn-evaluations-${turn.id}`, kind: "content", content: (
{turn.evaluations.map((evaluation) => ( ))}
), }); } }); return ; } function AutoTestToolDetails({ toolCalls, }: { toolCalls: BatchRunCase["turns"][number]["toolCalls"]; }) { return (
工具调用详情 {toolCalls.length} 次
{toolCalls.map((toolCall) => (
{toolCall.functionName} {toolCall.outcome === "success" ? "成功" : "错误"} ·{" "} {toolCall.durationMs} ms
                {toolCall.argumentsJson}
              
                {toolCall.resultJson}
              
))}
); } function EvaluationDetails({ evaluation, }: { evaluation: BatchEvaluationResult; }) { const passed = evaluation.status === "pass"; return (
{passed ? : }
{evaluation.label}
{passed ? "通过" : "未通过"}
评估标准
{evaluation.expected || "(未提供评估标准)"}
实际结果
{evaluation.actual || "(没有可评估内容)"}
{evaluation.reason && ( <>
判断说明
{evaluation.reason}
)}
); } function EmptyPanel({ icon, title, description, }: { icon: React.ReactNode; title: string; description: string; }) { return (
{icon}
{title}

{description}

); } function ErrorPanel({ title, description }: { title: string; description: string }) { return (
{title}

{description}

); } // ─── Bottom run bar ───────────────────────────────────────────────────────── export function AutoTestRunBar({ state, testCase, suites, cases, loading, onSelectCase, onClearCase, onStop, onRerun, }: { state: AutoTestRunState; testCase: TestCase | null; suites: TestSuite[]; cases: TestCase[]; loading: boolean; onSelectCase: (id: string) => void; onClearCase: () => void; onStop: () => void; onRerun: () => void; }) { const pickerTrigger = ( ); if (state.phase === "starting" || state.phase === "running") { return (
{testCase?.name} {state.stopping ? "停止中" : state.phase === "starting" ? "启动中" : `${state.currentTurn} / ${state.maxTurns}`}
); } if ( state.phase === "passed" || state.phase === "failed" || state.phase === "error" || state.phase === "cancelled" ) { const passed = state.phase === "passed"; const label = passed ? `PASS · ${state.currentTurn} 轮` : state.phase === "failed" ? "FAIL" : state.phase === "cancelled" ? "已停止" : "执行错误"; return (
{passed ? : } {testCase?.name} {label}
); } return (
{testCase && ( )}
{state.blockedMessage && testCase && (

{state.blockedMessage}

)}
); } // ─── Hook: real single-case run ───────────────────────────────────────────── export function useAutoTestRunner({ assistantId, enabled, hasUnsavedChanges, dynamicVariables, dynamicVariablesError, }: AutoTestRunnerOptions) { const [suites, setSuites] = useState([]); const [cases, setCases] = useState([]); const [evaluatorModels, setEvaluatorModels] = useState([]); const [evaluatorModelId, setEvaluatorModelId] = useState(""); const [catalogLoading, setCatalogLoading] = useState(true); const [catalogError, setCatalogError] = useState(""); const [selectedId, setSelectedId] = useState(null); const [run, setRun] = useState(null); const activeRunRef = useRef(null); const [starting, setStarting] = useState(false); const [stopping, setStopping] = useState(false); const [runError, setRunError] = useState(""); useEffect(() => { activeRunRef.current = run; }, [run]); useEffect( () => () => { const activeRun = activeRunRef.current; if (activeRun && !isRunFinished(activeRun)) { void batchRunsApi.cancel(activeRun.id); } }, [], ); useEffect(() => { if (!enabled) return; let disposed = false; void Promise.all([ testSuitesApi.list(), testCasesApi.list(), modelResourcesApi.list(), ]) .then(([loadedSuites, loadedCases, resources]) => { if (disposed) return; const llms = resources.filter( (item) => item.capability === "LLM" && item.enabled, ); setSuites(loadedSuites); setCases(loadedCases); setEvaluatorModels(llms); setCatalogError(""); const evaluator = llms.find((item) => item.isDefault) ?? llms[0]; setEvaluatorModelId(evaluator?.id ?? ""); }) .catch((error) => { if (disposed) return; setCatalogError( error instanceof Error ? error.message : "加载测试用例失败", ); }) .finally(() => { if (!disposed) setCatalogLoading(false); }); return () => { disposed = true; }; }, [enabled]); useEffect(() => { if (!enabled || !run || isRunFinished(run)) return; let disposed = false; let timer = 0; async function poll() { try { const snapshot = await batchRunsApi.get(run!.id); if (disposed) return; setRun(snapshot); setRunError(""); if (isRunFinished(snapshot)) { setStopping(false); return; } } catch (error) { if (!disposed) { setRunError( error instanceof Error ? error.message : "刷新测试结果失败", ); } } if (!disposed) timer = window.setTimeout(() => void poll(), RUN_POLL_MS); } timer = window.setTimeout(() => void poll(), RUN_POLL_MS); return () => { disposed = true; window.clearTimeout(timer); }; }, [enabled, run]); const selectedCase = useMemo( () => cases.find((item) => item.id === selectedId) ?? null, [cases, selectedId], ); const evaluator = useMemo( () => evaluatorModels.find((item) => item.id === evaluatorModelId) ?? null, [evaluatorModelId, evaluatorModels], ); const runCase = run?.cases[0] ?? null; let blockedMessage = ""; if (catalogLoading) blockedMessage = "正在加载测试用例…"; else if (catalogError) blockedMessage = catalogError; else if (!assistantId) blockedMessage = "请先保存助手,再开始自动测试"; else if (hasUnsavedChanges) blockedMessage = "请先保存当前改动,再开始自动测试"; else if (dynamicVariablesError) blockedMessage = dynamicVariablesError; else if (!selectedCase) blockedMessage = "请先选择测试用例"; else if (runnableMessage(selectedCase)) { blockedMessage = runnableMessage(selectedCase); } else if (!evaluatorModelId) blockedMessage = "请先配置可用的 LLM 评估模型"; let phase: AutoTestPhase = selectedCase ? "ready" : "idle"; if (starting) phase = "starting"; else if (run) { if (run.status === "cancelled" || runCase?.status === "skipped") { phase = "cancelled"; } else if (runCase?.status === "pass") phase = "passed"; else if (runCase?.status === "fail") phase = "failed"; else if (runCase?.status === "error") phase = "error"; else phase = "running"; } else if (runError) phase = "error"; const state: AutoTestRunState = { phase, selectedId, currentTurn: runCase?.turns.length ?? 0, maxTurns: selectedCase?.turns.length ?? 0, run, runCase, error: runError, stopping, blockedMessage, evaluatorName: evaluator?.name ?? "", }; const cancelActiveRun = useCallback(async () => { if (!run || isRunFinished(run)) return; setStopping(true); try { const snapshot = await batchRunsApi.cancel(run.id); setRun(snapshot); setRunError(""); if (isRunFinished(snapshot)) setStopping(false); } catch (error) { setRunError(error instanceof Error ? error.message : "停止测试失败"); setStopping(false); } }, [run]); const selectCase = useCallback( (id: string) => { if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id); setSelectedId(id); setRun(null); setRunError(""); setStarting(false); setStopping(false); }, [run], ); const clearCase = useCallback(() => { if (run && !isRunFinished(run)) void batchRunsApi.cancel(run.id); setSelectedId(null); setRun(null); setRunError(""); setStarting(false); setStopping(false); }, [run]); const start = useCallback(async () => { if ( !assistantId || !selectedCase || !evaluatorModelId || blockedMessage || starting ) { return; } setStarting(true); setRun(null); setRunError(""); try { const snapshot = await batchRunsApi.create({ assistantId, evaluatorModelResourceId: evaluatorModelId, caseIds: [selectedCase.id], dynamicVariables, title: `调试 · ${selectedCase.name}`, config: { concurrency: 1, timeoutSecs: 60, failureStrategy: "continue", errorRetryCount: 0, errorStrategy: "continue", }, }); setRun(snapshot); } catch (error) { setRunError(error instanceof Error ? error.message : "启动自动测试失败"); } finally { setStarting(false); } }, [ assistantId, blockedMessage, dynamicVariables, evaluatorModelId, selectedCase, starting, ]); const stop = useCallback(() => { if (run && !isRunFinished(run)) { void cancelActiveRun(); return; } setRun(null); setRunError(""); }, [cancelActiveRun, run]); const rerun = useCallback(() => { setRun(null); setRunError(""); window.setTimeout(() => void start(), 0); }, [start]); return { state, suites, cases, catalogLoading, selectedCase, selectCase, clearCase, start: () => void start(), stop, rerun, }; }