wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support image inputs in assistant tests

Browse Source
This commit is contained in:
Xin Wang
2026-08-23 23:46:18 +08:00
parent 0966f382b6
commit 9e4dfbe596
26 changed files with 1010 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/components/assistant-editor/debug-auto-test.tsx
+12 -1
View File
@@ -51,6 +51,7 @@ import {
import {
batchRunsApi,
modelResourcesApi,
testAssetsApi,
testCasesApi,
testSuitesApi,
type ModelResource,
@@ -101,7 +102,7 @@ function isRunFinished(run: BatchRunSnapshot): boolean {
function runnableMessage(item: TestCase): string {
if (item.inputMode !== "fixed_script_text") {
return "当前自动测试仅支持固定脚本 · 文字";
return "当前自动测试仅支持固定脚本 · 多模态";
}
return getTestCaseValidationMessage(item) ?? "";
}
@@ -653,6 +654,16 @@ function AutoTestTimeline({
role: "user",
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
content: turn.userInput,
attachments: turn.image
? [
{
id: turn.image.assetId,
type: "image",
url: testAssetsApi.contentUrl(turn.image.assetId),
alt: turn.image.filename,
},
]
: undefined,
},
});
items.push({
frontend/src/components/batch-test/batch-case-detail-drawer.tsx
+15 -4
View File
@@ -28,6 +28,7 @@ import {
type BatchToolCallRecord,
type BatchTurnResult,
} from "@/data/batch-run";
import { testAssetsApi } from "@/lib/api";
import { cn } from "@/lib/utils";
type BatchCaseDetailDrawerProps = {
@@ -98,7 +99,7 @@ export function BatchCaseDetailDrawer({
<section className="rounded-2xl border border-hairline bg-canvas-soft px-4 py-3.5">
<p className="text-xs font-medium text-foreground">执行契约</p>
<p className="mt-1 text-xs leading-5 text-muted-soft">
正在执行固定文字脚本；完成后将展示每一轮回复、工具调用和全部评估标准。
正在执行固定多模态脚本；完成后将展示每轮回复、工具调用和评估标准。
</p>
</section>
<ExecutionTimeline status={item.status} />
@@ -235,9 +236,19 @@ function TurnResultCard({ turn }: { turn: BatchTurnResult }) {
</p>
<div className="mt-2 grid gap-2 text-xs leading-5 sm:grid-cols-[72px_minmax(0,1fr)]">
<span className="text-muted-soft">用户输入</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.userInput}
</p>
<div className="space-y-2 text-muted-foreground">
{turn.image ? (
// eslint-disable-next-line @next/next/no-img-element
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="max-h-48 w-full max-w-xs rounded-xl object-cover"
/>
) : null}
{turn.userInput ? (
<p className="whitespace-pre-wrap">{turn.userInput}</p>
) : null}
</div>
<span className="text-muted-soft">Agent 回复</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.assistantReply}
frontend/src/components/pages/TestCasesPage.tsx
+5 -1
View File
@@ -112,6 +112,8 @@ function SuiteListView() {
}
useEffect(() => {
// Initial API hydration is intentionally owned by this mount effect.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reloadSuites();
}, []);
@@ -507,6 +509,8 @@ function SuiteDetailView({
}
useEffect(() => {
// Suite changes remount this view; hydrate the selected case once.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reload(initialCaseId ?? null);
// suiteId changes by remount; the initial request only runs once per view.
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -931,7 +935,7 @@ function SuiteDetailView({
{TEST_CASE_INPUT_MODE_LABEL[draft.inputMode]} · 开发中
</div>
<p className="max-w-sm text-xs leading-5 text-muted-foreground">
当前仅支持「固定脚本 · 文字」编辑器，其它输入模式稍后接入。
当前仅支持「固定脚本 · 多模态」编辑器，其它输入模式稍后接入。
</p>
</div>
)}
frontend/src/components/test-cases/input-mode-picker.tsx
+2 -2
View File
@@ -37,8 +37,8 @@ const FIXED_SCRIPT_OPTIONS: {
}[] = [
{
value: "fixed_script_text",
title: "文字",
description: "使用固定文字逐轮测试",
title: "文字与图片",
description: "使用固定文字或图片逐轮测试",
icon: Type,
available: true,
},
frontend/src/components/test-cases/next-reply-editor.tsx
+131 -4
View File
@@ -2,7 +2,7 @@
/**
* 测试用例编辑器主体：
* 对话上下文 → 固定文字脚本轮次 → 整体评估标准。
* 对话上下文 → 固定多模态脚本轮次 → 整体评估标准。
*/
import {
@@ -26,6 +26,8 @@ import {
ChevronDown,
ChevronUp,
GripVertical,
ImagePlus,
Loader2,
MessageSquareText,
MessagesSquare,
Plus,
@@ -71,12 +73,13 @@ import {
type KeywordMatchMode,
type OverallCriterion,
type ReplyExpectedBehavior,
type TestImageRef,
type TestCaseInputMode,
type ToolCallExpectedBehavior,
type ToolParamAssertion,
type ToolParamMatchMode,
} from "@/data/test-suites";
import { toolsApi, type Tool } from "@/lib/api";
import { testAssetsApi, toolsApi, type Tool } from "@/lib/api";
import { cn } from "@/lib/utils";
export type CaseEditorDraft = {
@@ -637,6 +640,9 @@ export function NextReplyEditorBody({
onChangeUserInput={(userInput) =>
updateTurn(turn.id, { userInput })
}
onChangeImage={(image) =>
updateTurn(turn.id, { image })
}
onAddReply={() =>
addBehavior(turn.id, createReplyBehavior("llm"))
}
@@ -812,6 +818,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds,
onToggleBehavior,
onChangeUserInput,
onChangeImage,
onAddReply,
onAddToolCall,
onUpdateBehavior,
@@ -825,6 +832,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds: Set<string>;
onToggleBehavior: (behaviorId: string) => void;
onChangeUserInput: (value: string) => void;
onChangeImage: (value: TestImageRef | null) => void;
onAddReply: () => void;
onAddToolCall: () => void;
onUpdateBehavior: (behavior: ExpectedBehavior) => void;
@@ -840,6 +848,34 @@ function SortableFixedInputTurn({
isDragging,
} = useSortable({ id: turn.id });
const [addOpen, setAddOpen] = useState(false);
const fileInputRef = useRef<HTMLInputElement>(null);
const [uploadingImage, setUploadingImage] = useState(false);
const [imageError, setImageError] = useState("");
async function uploadImage(file: File | undefined) {
if (!file || uploadingImage) return;
if (!file.type.startsWith("image/")) {
setImageError("请选择图片文件");
return;
}
setUploadingImage(true);
setImageError("");
try {
const asset = await testAssetsApi.uploadImage(file);
onChangeImage({
assetId: asset.id,
filename: asset.filename,
width: asset.width,
height: asset.height,
sizeBytes: asset.sizeBytes,
});
} catch (error) {
setImageError(error instanceof Error ? error.message : "图片上传失败");
} finally {
setUploadingImage(false);
if (fileInputRef.current) fileInputRef.current.value = "";
}
}
const style = {
transform: CSS.Transform.toString(transform),
@@ -888,17 +924,108 @@ function SortableFixedInputTurn({
</div>
<div className="space-y-3 pl-1">
<div className="space-y-1.5">
<div
className="space-y-1.5"
onDragOver={(event) => {
if (event.dataTransfer.types.includes("Files")) event.preventDefault();
}}
onDrop={(event) => {
const file = Array.from(event.dataTransfer.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
>
<div className="text-xs font-medium text-muted-foreground">
用户输入
</div>
<Textarea
value={turn.userInput}
onChange={(event) => onChangeUserInput(event.target.value)}
placeholder="输入本轮用户原话…"
onPaste={(event) => {
const file = Array.from(event.clipboardData.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
placeholder={
turn.image
? "可选：补充用户与图片一起提交的问题…"
: "输入本轮用户原话，或添加一张图片…"
}
rows={2}
className="field-sizing-fixed min-h-[56px] resize-y border-hairline-strong bg-background text-sm"
/>
{turn.image ? (
<div className="flex items-center gap-3 rounded-xl border border-hairline bg-canvas-soft p-2">
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="h-16 w-20 shrink-0 rounded-lg object-cover"
/>
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium text-foreground">
{turn.image.filename}
</div>
<div className="mt-1 text-[11px] text-muted-soft">
{turn.image.width} × {turn.image.height}
{turn.image.sizeBytes > 0
? ` · ${Math.max(1, Math.round(turn.image.sizeBytes / 1024))} KB`
: ""}
</div>
</div>
<Button
type="button"
variant="ghost"
size="icon-sm"
className="shrink-0 text-muted-soft hover:text-destructive"
onClick={() => {
setImageError("");
onChangeImage(null);
}}
aria-label="移除本轮图片"
>
<Trash2 size={14} />
</Button>
</div>
) : null}
<div className="flex flex-wrap items-center gap-2">
<input
ref={fileInputRef}
type="file"
accept="image/*"
className="sr-only"
onChange={(event) => void uploadImage(event.target.files?.[0])}
/>
<Button
type="button"
variant="outline"
size="sm"
className="h-8 gap-1.5 rounded-full border-hairline-strong text-xs text-muted-foreground"
disabled={uploadingImage}
onClick={() => fileInputRef.current?.click()}
>
{uploadingImage ? (
<Loader2 size={13} className="animate-spin" />
) : (
<ImagePlus size={13} />
)}
{turn.image ? "替换图片" : "添加图片"}
</Button>
<span className="text-[11px] text-muted-soft">
支持选择、粘贴或拖入；每轮最多一张
</span>
</div>
{imageError ? (
<p className="text-[11px] text-destructive" role="alert">
{imageError}
</p>
) : null}
</div>
<div className="space-y-2">