wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support image inputs in assistant tests

Browse Source
This commit is contained in:
Xin Wang
2026-08-23 23:46:18 +08:00
parent 0966f382b6
commit 9e4dfbe596
26 changed files with 1010 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/components/assistant-editor/debug-auto-test.tsx
+12 -1
View File
@@ -51,6 +51,7 @@ import {
import {
batchRunsApi,
modelResourcesApi,
testAssetsApi,
testCasesApi,
testSuitesApi,
type ModelResource,
@@ -101,7 +102,7 @@ function isRunFinished(run: BatchRunSnapshot): boolean {
function runnableMessage(item: TestCase): string {
if (item.inputMode !== "fixed_script_text") {
return "当前自动测试仅支持固定脚本 · 文字";
return "当前自动测试仅支持固定脚本 · 多模态";
}
return getTestCaseValidationMessage(item) ?? "";
}
@@ -653,6 +654,16 @@ function AutoTestTimeline({
role: "user",
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
content: turn.userInput,
attachments: turn.image
? [
{
id: turn.image.assetId,
type: "image",
url: testAssetsApi.contentUrl(turn.image.assetId),
alt: turn.image.filename,
},
]
: undefined,
},
});
items.push({
frontend/src/components/batch-test/batch-case-detail-drawer.tsx
+15 -4
View File
@@ -28,6 +28,7 @@ import {
type BatchToolCallRecord,
type BatchTurnResult,
} from "@/data/batch-run";
import { testAssetsApi } from "@/lib/api";
import { cn } from "@/lib/utils";
type BatchCaseDetailDrawerProps = {
@@ -98,7 +99,7 @@ export function BatchCaseDetailDrawer({
<section className="rounded-2xl border border-hairline bg-canvas-soft px-4 py-3.5">
<p className="text-xs font-medium text-foreground">执行契约</p>
<p className="mt-1 text-xs leading-5 text-muted-soft">
正在执行固定文字脚本；完成后将展示每一轮回复、工具调用和全部评估标准。
正在执行固定多模态脚本；完成后将展示每轮回复、工具调用和评估标准。
</p>
</section>
<ExecutionTimeline status={item.status} />
@@ -235,9 +236,19 @@ function TurnResultCard({ turn }: { turn: BatchTurnResult }) {
</p>
<div className="mt-2 grid gap-2 text-xs leading-5 sm:grid-cols-[72px_minmax(0,1fr)]">
<span className="text-muted-soft">用户输入</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.userInput}
</p>
<div className="space-y-2 text-muted-foreground">
{turn.image ? (
// eslint-disable-next-line @next/next/no-img-element
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="max-h-48 w-full max-w-xs rounded-xl object-cover"
/>
) : null}
{turn.userInput ? (
<p className="whitespace-pre-wrap">{turn.userInput}</p>
) : null}
</div>
<span className="text-muted-soft">Agent 回复</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.assistantReply}
frontend/src/components/pages/TestCasesPage.tsx
+5 -1
View File
@@ -112,6 +112,8 @@ function SuiteListView() {
}
useEffect(() => {
// Initial API hydration is intentionally owned by this mount effect.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reloadSuites();
}, []);
@@ -507,6 +509,8 @@ function SuiteDetailView({
}
useEffect(() => {
// Suite changes remount this view; hydrate the selected case once.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reload(initialCaseId ?? null);
// suiteId changes by remount; the initial request only runs once per view.
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -931,7 +935,7 @@ function SuiteDetailView({
{TEST_CASE_INPUT_MODE_LABEL[draft.inputMode]} · 开发中
</div>
<p className="max-w-sm text-xs leading-5 text-muted-foreground">
当前仅支持「固定脚本 · 文字」编辑器，其它输入模式稍后接入。
当前仅支持「固定脚本 · 多模态」编辑器，其它输入模式稍后接入。
</p>
</div>
)}
frontend/src/components/test-cases/input-mode-picker.tsx
+2 -2
View File
@@ -37,8 +37,8 @@ const FIXED_SCRIPT_OPTIONS: {
}[] = [
{
value: "fixed_script_text",
title: "文字",
description: "使用固定文字逐轮测试",
title: "文字与图片",
description: "使用固定文字或图片逐轮测试",
icon: Type,
available: true,
},
frontend/src/components/test-cases/next-reply-editor.tsx
+131 -4
View File
@@ -2,7 +2,7 @@
/**
* 测试用例编辑器主体：
* 对话上下文 → 固定文字脚本轮次 → 整体评估标准。
* 对话上下文 → 固定多模态脚本轮次 → 整体评估标准。
*/
import {
@@ -26,6 +26,8 @@ import {
ChevronDown,
ChevronUp,
GripVertical,
ImagePlus,
Loader2,
MessageSquareText,
MessagesSquare,
Plus,
@@ -71,12 +73,13 @@ import {
type KeywordMatchMode,
type OverallCriterion,
type ReplyExpectedBehavior,
type TestImageRef,
type TestCaseInputMode,
type ToolCallExpectedBehavior,
type ToolParamAssertion,
type ToolParamMatchMode,
} from "@/data/test-suites";
import { toolsApi, type Tool } from "@/lib/api";
import { testAssetsApi, toolsApi, type Tool } from "@/lib/api";
import { cn } from "@/lib/utils";
export type CaseEditorDraft = {
@@ -637,6 +640,9 @@ export function NextReplyEditorBody({
onChangeUserInput={(userInput) =>
updateTurn(turn.id, { userInput })
}
onChangeImage={(image) =>
updateTurn(turn.id, { image })
}
onAddReply={() =>
addBehavior(turn.id, createReplyBehavior("llm"))
}
@@ -812,6 +818,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds,
onToggleBehavior,
onChangeUserInput,
onChangeImage,
onAddReply,
onAddToolCall,
onUpdateBehavior,
@@ -825,6 +832,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds: Set<string>;
onToggleBehavior: (behaviorId: string) => void;
onChangeUserInput: (value: string) => void;
onChangeImage: (value: TestImageRef | null) => void;
onAddReply: () => void;
onAddToolCall: () => void;
onUpdateBehavior: (behavior: ExpectedBehavior) => void;
@@ -840,6 +848,34 @@ function SortableFixedInputTurn({
isDragging,
} = useSortable({ id: turn.id });
const [addOpen, setAddOpen] = useState(false);
const fileInputRef = useRef<HTMLInputElement>(null);
const [uploadingImage, setUploadingImage] = useState(false);
const [imageError, setImageError] = useState("");
async function uploadImage(file: File | undefined) {
if (!file || uploadingImage) return;
if (!file.type.startsWith("image/")) {
setImageError("请选择图片文件");
return;
}
setUploadingImage(true);
setImageError("");
try {
const asset = await testAssetsApi.uploadImage(file);
onChangeImage({
assetId: asset.id,
filename: asset.filename,
width: asset.width,
height: asset.height,
sizeBytes: asset.sizeBytes,
});
} catch (error) {
setImageError(error instanceof Error ? error.message : "图片上传失败");
} finally {
setUploadingImage(false);
if (fileInputRef.current) fileInputRef.current.value = "";
}
}
const style = {
transform: CSS.Transform.toString(transform),
@@ -888,17 +924,108 @@ function SortableFixedInputTurn({
</div>
<div className="space-y-3 pl-1">
<div className="space-y-1.5">
<div
className="space-y-1.5"
onDragOver={(event) => {
if (event.dataTransfer.types.includes("Files")) event.preventDefault();
}}
onDrop={(event) => {
const file = Array.from(event.dataTransfer.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
>
<div className="text-xs font-medium text-muted-foreground">
用户输入
</div>
<Textarea
value={turn.userInput}
onChange={(event) => onChangeUserInput(event.target.value)}
placeholder="输入本轮用户原话…"
onPaste={(event) => {
const file = Array.from(event.clipboardData.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
placeholder={
turn.image
? "可选：补充用户与图片一起提交的问题…"
: "输入本轮用户原话，或添加一张图片…"
}
rows={2}
className="field-sizing-fixed min-h-[56px] resize-y border-hairline-strong bg-background text-sm"
/>
{turn.image ? (
<div className="flex items-center gap-3 rounded-xl border border-hairline bg-canvas-soft p-2">
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="h-16 w-20 shrink-0 rounded-lg object-cover"
/>
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium text-foreground">
{turn.image.filename}
</div>
<div className="mt-1 text-[11px] text-muted-soft">
{turn.image.width} × {turn.image.height}
{turn.image.sizeBytes > 0
? ` · ${Math.max(1, Math.round(turn.image.sizeBytes / 1024))} KB`
: ""}
</div>
</div>
<Button
type="button"
variant="ghost"
size="icon-sm"
className="shrink-0 text-muted-soft hover:text-destructive"
onClick={() => {
setImageError("");
onChangeImage(null);
}}
aria-label="移除本轮图片"
>
<Trash2 size={14} />
</Button>
</div>
) : null}
<div className="flex flex-wrap items-center gap-2">
<input
ref={fileInputRef}
type="file"
accept="image/*"
className="sr-only"
onChange={(event) => void uploadImage(event.target.files?.[0])}
/>
<Button
type="button"
variant="outline"
size="sm"
className="h-8 gap-1.5 rounded-full border-hairline-strong text-xs text-muted-foreground"
disabled={uploadingImage}
onClick={() => fileInputRef.current?.click()}
>
{uploadingImage ? (
<Loader2 size={13} className="animate-spin" />
) : (
<ImagePlus size={13} />
)}
{turn.image ? "替换图片" : "添加图片"}
</Button>
<span className="text-[11px] text-muted-soft">
支持选择、粘贴或拖入；每轮最多一张
</span>
</div>
{imageError ? (
<p className="text-[11px] text-destructive" role="alert">
{imageError}
</p>
) : null}
</div>
<div className="space-y-2">
frontend/src/data/batch-run.ts
+7
View File
@@ -54,6 +54,13 @@ export type BatchTurnResult = {
id: string;
index: number;
userInput: string;
image: {
assetId: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
} | null;
assistantReply: string;
toolCalls: BatchToolCallRecord[];
runtimeRecords?: Array<Record<string, unknown>>;
frontend/src/data/test-suites.ts
+17 -6
View File
@@ -1,7 +1,7 @@
/**
* 测试集 / 测试用例的前后端共享契约与编辑器辅助函数。
* 两层：Test Suite → Test Case。
* MVP 仅运行固定文字脚本；其它输入模式保留为产品路线提示。
* MVP 运行固定文字/图片脚本；其它输入模式保留为产品路线提示。
*/
/** 顶部模式选择器的唯一来源；不再单独维护「测试类型」字段 */
@@ -80,6 +80,14 @@ export type ToolCallExpectedBehavior = {
export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
export type TestImageRef = {
assetId: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
};
/**
* 固定输入的一轮：只编辑 User 输入；Agent 回复 / Tool Call 由运行时生成。
* behaviors 为 0~N 个预期行为（AND）；空数组表示仅推动对话、不做断言。
@@ -87,6 +95,7 @@ export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
export type FixedInputTurn = {
id: string;
userInput: string;
image: TestImageRef | null;
behaviors: ExpectedBehavior[];
};
@@ -134,7 +143,7 @@ export const DEFAULT_INPUT_MODE: TestCaseInputMode = "fixed_script_text";
/**
* MVP 可运行性规则。
* 目前执行器只支持固定文字；每一轮都需要用户输入，并至少配置一项断言。
* 目前执行器支持固定文字和单张图片；每轮至少有一种输入，并配置断言。
*/
export function getTestCaseValidationMessage(
item: Pick<
@@ -219,10 +228,10 @@ export function getTestCaseValidationMessage(
.join("、")}`;
}
const emptyTurnIndex = item.turns.findIndex(
(turn) => !turn.userInput.trim(),
(turn) => !turn.userInput.trim() && !turn.image,
);
if (emptyTurnIndex >= 0) {
return `第 ${emptyTurnIndex + 1} 轮用户输入不能为空`;
return `第 ${emptyTurnIndex + 1} 轮需要填写用户文字或添加图片`;
}
for (let turnIndex = 0; turnIndex < item.turns.length; turnIndex += 1) {
const turn = item.turns[turnIndex];
@@ -267,7 +276,7 @@ export function isTestCaseRunnable(
}
export const TEST_CASE_INPUT_MODE_LABEL: Record<TestCaseInputMode, string> = {
fixed_script_text: "固定脚本 · 文字",
fixed_script_text: "固定脚本 · 多模态",
fixed_script_turn_voice: "固定脚本 · 逐轮语音",
fixed_script_continuous_voice: "固定脚本 · 连续语音",
user_sim_text: "用户模拟 · 文字",
@@ -279,7 +288,7 @@ export const TEST_CASE_INPUT_MODE_SHORT_LABEL: Record<
TestCaseInputMode,
string
> = {
fixed_script_text: "固定文字",
fixed_script_text: "固定输入",
fixed_script_turn_voice: "逐轮语音",
fixed_script_continuous_voice: "连续语音",
user_sim_text: "模拟文字",
@@ -412,6 +421,7 @@ export function createEmptyFixedInputTurn(
return {
id: newEditorId("turn"),
userInput,
image: null,
behaviors: [],
};
}
@@ -484,6 +494,7 @@ export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: turn.id,
userInput: turn.userInput,
image: turn.image ? { ...turn.image } : null,
behaviors: turn.behaviors.map(cloneBehavior),
}));
}
frontend/src/lib/api.ts
+27
View File
@@ -854,6 +854,33 @@ export type TestCaseWrite = {
overallCriteria: OverallCriterion[];
};
export type TestImageAsset = {
id: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
contentUrl: string;
};
export const testAssetsApi = {
uploadImage: (file: File) => {
const body = new FormData();
body.append("file", file);
return request<TestImageAsset>("/api/test-assets/images", {
method: "POST",
body,
});
},
contentUrl: (assetId: string) =>
`${API_BASE}/api/test-assets/${encodeURIComponent(assetId)}/content`,
remove: (assetId: string) =>
request<{ ok: boolean }>(
`/api/test-assets/${encodeURIComponent(assetId)}`,
{ method: "DELETE" },
),
};
export const testSuitesApi = {
list: () => request<TestSuite[]>("/api/test-suites"),
get: (id: string) => request<TestSuite>(`/api/test-suites/${id}`),