wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: support image inputs in assistant tests

Browse Source
This commit is contained in:
Xin Wang
2026-08-23 23:46:18 +08:00
parent 0966f382b6
commit 9e4dfbe596
26 changed files with 1010 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app.py
+2
View File
@@ -48,6 +48,7 @@ from routes import (
realtime_api_keys,
site_settings,
test_cases,
test_assets,
test_runs,
tools,
voice_webrtc,
@@ -104,6 +105,7 @@ app.include_router(openai_realtime.router)
app.include_router(realtime_api_keys.router)
app.include_router(site_settings.router)
app.include_router(test_cases.router)
app.include_router(test_assets.router)
app.include_router(test_runs.router)
app.include_router(tools.router)
app.include_router(voice_webrtc.router)
backend/db/models.py
+17
View File
@@ -475,6 +475,23 @@ class TestSuite(Base):
)
class TestAsset(Base):
"""Immutable image fixture referenced by one or more test definitions."""
__tablename__ = "test_assets"
id: Mapped[str] = mapped_column(String(40), primary_key=True)
original_name: Mapped[str] = mapped_column(String(255), default="image.jpg")
storage_key: Mapped[str] = mapped_column(String(1024), unique=True)
mime_type: Mapped[str] = mapped_column(String(128), default="image/jpeg")
width: Mapped[int] = mapped_column(Integer)
height: Mapped[int] = mapped_column(Integer)
size_bytes: Mapped[int] = mapped_column(Integer)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
class TestCase(Base):
"""One editable test definition; nested turns remain one JSON document."""
backend/migrations/versions/20260818_0023_add_test_assets.py
+53
View File
@@ -0,0 +1,53 @@
"""add persistent image fixtures for test cases
Revision ID: 20260818_0023
Revises: 20260813_0022
"""
from __future__ import annotations
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "20260818_0023"
down_revision: str | Sequence[str] | None = "20260813_0022"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"test_assets",
sa.Column("id", sa.String(length=40), nullable=False),
sa.Column(
"original_name",
sa.String(length=255),
server_default="image.jpg",
nullable=False,
),
sa.Column("storage_key", sa.String(length=1024), nullable=False),
sa.Column(
"mime_type",
sa.String(length=128),
server_default="image/jpeg",
nullable=False,
),
sa.Column("width", sa.Integer(), nullable=False),
sa.Column("height", sa.Integer(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=False),
sa.Column(
"created_at",
sa.DateTime(timezone=True),
server_default=sa.text("now()"),
nullable=False,
),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("storage_key"),
)
def downgrade() -> None:
op.drop_table("test_assets")
backend/routes/test_assets.py
+172
View File
@@ -0,0 +1,172 @@
"""Persistent image fixtures for repeatable assistant tests."""
from __future__ import annotations
import asyncio
from pathlib import Path
from uuid import uuid4
from db.models import TestAsset, TestCase, TestRunCase
from db.session import get_session
from fastapi import APIRouter, Depends, File, HTTPException, UploadFile
from fastapi.responses import Response
from loguru import logger
from pydantic import BaseModel
from services.auth import require_admin
from services.test_assets import delete_test_image, read_test_image, store_test_image
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
import settings
router = APIRouter(
prefix="/api/test-assets",
tags=["test-assets"],
dependencies=[Depends(require_admin)],
)
class TestImageAssetOut(BaseModel):
id: str
filename: str
width: int
height: int
sizeBytes: int
contentUrl: str
def _new_id() -> str:
return f"ta_{uuid4().hex}"
def _safe_filename(value: str | None) -> str:
name = Path(value or "image.jpg").name.strip()
return (name or "image.jpg")[:255]
def _out(row: TestAsset) -> TestImageAssetOut:
return TestImageAssetOut(
id=row.id,
filename=row.original_name,
width=row.width,
height=row.height,
sizeBytes=row.size_bytes,
contentUrl=f"/api/test-assets/{row.id}/content",
)
def _contains_asset_id(value: object, asset_id: str) -> bool:
if isinstance(value, dict):
if value.get("assetId") == asset_id or value.get("asset_id") == asset_id:
return True
return any(_contains_asset_id(item, asset_id) for item in value.values())
if isinstance(value, list):
return any(_contains_asset_id(item, asset_id) for item in value)
return False
async def _is_referenced(session: AsyncSession, asset_id: str) -> bool:
definitions = (
await session.execute(select(TestCase.definition))
).scalars().all()
if any(_contains_asset_id(value, asset_id) for value in definitions):
return True
snapshots = (
await session.execute(select(TestRunCase.case_snapshot))
).scalars().all()
return any(_contains_asset_id(value, asset_id) for value in snapshots)
@router.post("/images", response_model=TestImageAssetOut)
async def upload_test_image(
file: UploadFile = File(...),
session: AsyncSession = Depends(get_session),
):
data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1)
if len(data) > settings.INPUT_IMAGE_MAX_BYTES:
limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024
raise HTTPException(413, f"图片不能超过 {limit_mb} MB")
if not data:
raise HTTPException(400, "图片文件不能为空")
asset_id = _new_id()
try:
stored = await asyncio.to_thread(store_test_image, asset_id, data)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(503, "图片暂时无法上传，请确认对象存储可用") from exc
row = TestAsset(
id=asset_id,
original_name=_safe_filename(file.filename),
storage_key=stored.storage_key,
mime_type=stored.mime_type,
width=stored.width,
height=stored.height,
size_bytes=stored.size_bytes,
)
session.add(row)
try:
await session.commit()
except Exception:
await session.rollback()
try:
await asyncio.to_thread(delete_test_image, stored.storage_key)
except Exception:
logger.warning(f"回滚测试图片上传时清理对象失败: {stored.storage_key}")
raise
await session.refresh(row)
return _out(row)
@router.get("/{asset_id}", response_model=TestImageAssetOut)
async def get_test_asset(
asset_id: str,
session: AsyncSession = Depends(get_session),
):
row = await session.get(TestAsset, asset_id)
if row is None:
raise HTTPException(404, "测试图片不存在")
return _out(row)
@router.get("/{asset_id}/content")
async def get_test_asset_content(
asset_id: str,
session: AsyncSession = Depends(get_session),
):
row = await session.get(TestAsset, asset_id)
if row is None:
raise HTTPException(404, "测试图片不存在")
try:
data = await asyncio.to_thread(read_test_image, row.storage_key)
except Exception as exc:
raise HTTPException(503, "测试图片暂时无法读取") from exc
return Response(
content=data,
media_type=row.mime_type,
headers={"Cache-Control": "private, max-age=3600"},
)
@router.delete("/{asset_id}")
async def delete_test_asset(
asset_id: str,
session: AsyncSession = Depends(get_session),
):
row = await session.get(TestAsset, asset_id)
if row is None:
raise HTTPException(404, "测试图片不存在")
if await _is_referenced(session, asset_id):
raise HTTPException(409, "测试图片仍被用例或运行记录引用")
storage_key = row.storage_key
await session.delete(row)
await session.commit()
try:
await asyncio.to_thread(delete_test_image, storage_key)
except Exception:
logger.warning(f"测试图片数据已删除，但对象存储清理失败: {storage_key}")
return {"ok": True}
backend/routes/test_cases.py
+16 -9
View File
@@ -9,6 +9,7 @@ from db.models import TestCase, TestRunCase, TestSuite
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException, Query
from services.auth import require_admin
from services.test_case_assets import canonicalize_test_assets
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from test_schemas import (
@@ -79,13 +80,6 @@ def _case_out(row: TestCase) -> TestCaseOut:
)
def _apply_case_write(row: TestCase, body: TestCaseWrite) -> None:
row.name = body.name.strip()
row.description = body.description.strip()
row.input_mode = body.input_mode
row.definition = body.definition().model_dump(mode="json", by_alias=True)
async def _renumber_cases(session: AsyncSession, suite_id: str) -> None:
rows = (
await session.execute(
@@ -232,13 +226,18 @@ async def create_test_case(
max_order = await session.scalar(
select(func.max(TestCase.sort_order)).where(TestCase.suite_id == suite_id)
)
definition = body.definition()
try:
await canonicalize_test_assets(session, definition)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
row = TestCase(
id=_new_id("tc"),
suite_id=suite_id,
name=body.name.strip(),
description=body.description.strip(),
input_mode=body.input_mode,
definition=body.definition().model_dump(mode="json", by_alias=True),
definition=definition.model_dump(mode="json", by_alias=True),
sort_order=int(max_order if max_order is not None else -1) + 1,
last_result="not_run",
)
@@ -315,7 +314,15 @@ async def update_test_case(
row = await session.get(TestCase, case_id)
if row is None:
raise HTTPException(404, "测试用例不存在")
_apply_case_write(row, body)
definition = body.definition()
try:
await canonicalize_test_assets(session, definition)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
row.name = body.name.strip()
row.description = body.description.strip()
row.input_mode = body.input_mode
row.definition = definition.model_dump(mode="json", by_alias=True)
await _touch_suite(session, row.suite_id)
await session.commit()
await session.refresh(row)
backend/routes/test_runs.py
+17 -2
View File
@@ -18,6 +18,8 @@ from fastapi import APIRouter, Depends, HTTPException
from services.auth import require_admin
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import DynamicVariableError, prepare_dynamic_config
from services.realtime.launcher import validate_visual_runtime
from services.test_case_assets import canonicalize_test_assets, image_asset_ids
from services.test_runs.orchestrator import test_run_orchestrator
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -140,13 +142,18 @@ async def create_test_run(
raise HTTPException(422, "测试用例不存在: " + "、".join(missing))
ordered_cases = [by_id[case_id] for case_id in body.case_ids]
tools_by_id = {tool.id: tool for tool in runtime_config.tools}
definitions_by_id: dict[str, TestCaseDefinition] = {}
has_image_input = False
for row in ordered_cases:
if row.input_mode != "fixed_script_text":
raise HTTPException(422, f"用例“{row.name}”不是可运行的固定文字模式")
raise HTTPException(422, f"用例“{row.name}”不是可运行的固定多模态模式")
try:
definition = TestCaseDefinition.model_validate(row.definition or {})
await canonicalize_test_assets(session, definition)
except ValueError as exc:
raise HTTPException(422, f"用例“{row.name}”校验失败: {exc}") from exc
definitions_by_id[row.id] = definition
has_image_input = has_image_input or bool(image_asset_ids(definition))
for turn in definition.turns:
for behavior in turn.behaviors:
if behavior.type != "tool_call":
@@ -159,6 +166,14 @@ async def create_test_run(
"不属于当前被测助手",
)
if has_image_input:
try:
vision_enabled = validate_visual_runtime(runtime_config)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
if not vision_enabled:
raise HTTPException(422, "测试用例包含图片，但当前助手未开启视觉能力")
suite_ids = list(dict.fromkeys(row.suite_id for row in ordered_cases))
suites = (
await session.execute(select(TestSuite).where(TestSuite.id.in_(suite_ids)))
@@ -192,7 +207,7 @@ async def create_test_run(
)
session.add(run)
for position, row in enumerate(ordered_cases):
definition = TestCaseDefinition.model_validate(row.definition or {})
definition = definitions_by_id[row.id]
snapshot = {
"id": row.id,
"suiteId": row.suite_id,
backend/services/image_processing.py
+41
View File
@@ -0,0 +1,41 @@
"""Shared validation and normalization for user-supplied images."""
from __future__ import annotations
from io import BytesIO
from PIL import Image, ImageOps, UnidentifiedImageError
import settings
def normalize_image(data: bytes) -> tuple[bytes, int, int]:
"""Return a bounded, orientation-corrected JPEG safe for model input."""
try:
source = Image.open(BytesIO(data))
width, height = source.size
if width <= 0 or height <= 0:
raise ValueError("图片尺寸无效")
if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
raise ValueError("图片像素尺寸过大")
source.load()
except (UnidentifiedImageError, OSError) as exc:
raise ValueError("文件不是可识别的图片") from exc
image = ImageOps.exif_transpose(source)
image.thumbnail(
(settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
Image.Resampling.LANCZOS,
)
if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
image = background
else:
image = image.convert("RGB")
buffer = BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
return buffer.getvalue(), image.width, image.height
backend/services/input_assets.py
+2 -34
View File
@@ -8,12 +8,10 @@ import hmac
import json
import time
from dataclasses import dataclass
from io import BytesIO
from uuid import uuid4
from PIL import Image, ImageOps, UnidentifiedImageError
import settings
from services.image_processing import normalize_image
from services.object_storage import delete_object, get_object, put_object
@@ -46,38 +44,8 @@ def _signature(payload: str) -> str:
return _b64encode(digest)
def _normalize_image(data: bytes) -> tuple[bytes, int, int]:
try:
source = Image.open(BytesIO(data))
width, height = source.size
if width <= 0 or height <= 0:
raise ValueError("图片尺寸无效")
if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
raise ValueError("图片像素尺寸过大")
source.load()
except (UnidentifiedImageError, OSError) as exc:
raise ValueError("文件不是可识别的图片") from exc
image = ImageOps.exif_transpose(source)
image.thumbnail(
(settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
Image.Resampling.LANCZOS,
)
if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
rgba = image.convert("RGBA")
background = Image.new("RGB", rgba.size, "white")
background.paste(rgba, mask=rgba.getchannel("A"))
image = background
else:
image = image.convert("RGB")
buffer = BytesIO()
image.save(buffer, format="JPEG", quality=85, optimize=True)
return buffer.getvalue(), image.width, image.height
def store_input_image(data: bytes) -> StoredInputImage:
normalized, width, height = _normalize_image(data)
normalized, width, height = normalize_image(data)
key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg"
put_object(key, normalized, "image/jpeg")
expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS
backend/services/pipecat/pipeline.py
+7 -53
View File
@@ -34,12 +34,11 @@ from services.tool_policy import policy_for_tool
from services.vision import (
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_main_llm_as_vision,
config_with_vision_resource,
image_data_uri,
image_frame_from_jpeg,
image_jpeg_bytes,
resolve_vision_question,
resolve_vision_input,
vision_system_hint,
vision_tool_schema_parts,
)
@@ -151,30 +150,6 @@ def _require(value: str, label: str) -> str:
raise ValueError(f"缺少模型资源配置: {label}")
def _vision_uses_main_llm(cfg: AssistantConfig) -> bool:
"""模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。"""
return not cfg.vision_model_resource_id and cfg.llm_support_image_input
def _workflow_vision_uses_main_llm(
cfg: AssistantConfig,
scope: dict[str, Any],
) -> bool:
"""Resolve native versus auxiliary vision for the active Workflow Agent."""
if not scope.get("enabled"):
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
if scope.get("vision_model_resource_id"):
return False
llm_resource_id = str(scope.get("llm_resource_id") or "")
resource = cfg.workflow_model_resources.get(llm_resource_id)
if not resource:
raise ValueError(f"当前 Workflow Agent 的 LLM 资源未加载:{llm_resource_id}")
if not resource.support_image_input:
raise ValueError("当前 Workflow Agent 的 LLM 不支持图片输入")
return True
def _multimodal_user_input_frame(
image_frame: UserImageRawFrame,
prompt_text: str,
@@ -485,7 +460,7 @@ async def run_pipeline(
vision_native_mode = (
vision_enabled
and cfg.type != "workflow"
and _vision_uses_main_llm(cfg)
and resolve_vision_input(cfg).native
)
vision_state: dict[str, str | None] = {"client_id": None}
workflow_vision_scope: dict[str, Any] = {
@@ -497,9 +472,9 @@ async def run_pipeline(
}
def active_vision_uses_main_llm() -> bool:
if cfg.type == "workflow":
return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope)
return vision_native_mode
if cfg.type != "workflow":
return vision_native_mode
return resolve_vision_input(cfg, workflow_vision_scope).native
vision_description, vision_properties, vision_required = vision_tool_schema_parts(
cfg.vision_prompt_mode,
cfg.vision_prompt,
@@ -609,29 +584,8 @@ async def run_pipeline(
workflow_vision_function = None
def active_vision_config() -> AssistantConfig:
if cfg.type != "workflow":
return cfg
if not workflow_vision_scope.get("enabled"):
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
vision_resource_id = str(
workflow_vision_scope.get("vision_model_resource_id") or ""
)
llm_resource_id = str(workflow_vision_scope.get("llm_resource_id") or "")
resource_id = vision_resource_id or llm_resource_id
resource = cfg.workflow_model_resources.get(resource_id)
if resource:
result = config_with_vision_resource(cfg, resource)
elif not vision_resource_id:
result = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
result.vision_prompt_mode = str(
workflow_vision_scope.get("vision_prompt_mode") or "runtime"
)
result.vision_prompt = str(
workflow_vision_scope.get("vision_prompt") or ""
).strip()
return result
scope = workflow_vision_scope if cfg.type == "workflow" else None
return resolve_vision_input(cfg, scope).config
if cfg.type == "workflow" and vision_enabled:
async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:
backend/services/test_assets.py
+44
View File
@@ -0,0 +1,44 @@
"""Persistent image fixtures used by repeatable assistant test cases."""
from __future__ import annotations
from dataclasses import dataclass
from services.image_processing import normalize_image
from services.object_storage import delete_object, get_object, put_object
TEST_ASSET_PREFIX = "test-assets/"
@dataclass(frozen=True)
class StoredTestImage:
storage_key: str
width: int
height: int
size_bytes: int
mime_type: str = "image/jpeg"
def store_test_image(asset_id: str, data: bytes) -> StoredTestImage:
normalized, width, height = normalize_image(data)
key = f"{TEST_ASSET_PREFIX}{asset_id}.jpg"
put_object(key, normalized, "image/jpeg")
return StoredTestImage(
storage_key=key,
width=width,
height=height,
size_bytes=len(normalized),
)
def read_test_image(storage_key: str) -> bytes:
if not storage_key.startswith(TEST_ASSET_PREFIX):
raise ValueError("测试图片存储位置无效")
return get_object(storage_key)
def delete_test_image(storage_key: str) -> None:
if not storage_key.startswith(TEST_ASSET_PREFIX):
raise ValueError("测试图片存储位置无效")
delete_object(storage_key)
backend/services/test_case_assets.py
+72
View File
@@ -0,0 +1,72 @@
"""Validation and loading helpers for image fixtures in test definitions."""
from __future__ import annotations
import asyncio
from db.models import TestAsset
from services.test_assets import read_test_image
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from test_schemas import TestCaseDefinition
def image_asset_ids(definition: TestCaseDefinition) -> set[str]:
return {
turn.image.asset_id
for turn in definition.turns
if turn.image is not None
}
async def canonicalize_test_assets(
session: AsyncSession,
definition: TestCaseDefinition,
) -> None:
"""Validate references and replace client metadata with authoritative values."""
asset_ids = image_asset_ids(definition)
if not asset_ids:
return
rows = (
await session.execute(select(TestAsset).where(TestAsset.id.in_(asset_ids)))
).scalars().all()
by_id = {row.id: row for row in rows}
missing = sorted(asset_ids - by_id.keys())
if missing:
raise ValueError("测试图片不存在: " + "、".join(missing))
for turn in definition.turns:
if turn.image is None:
continue
row = by_id[turn.image.asset_id]
turn.image.filename = row.original_name
turn.image.width = row.width
turn.image.height = row.height
turn.image.size_bytes = row.size_bytes
async def load_test_asset_bytes(
session: AsyncSession,
definition: TestCaseDefinition,
) -> dict[str, bytes]:
"""Load immutable image fixtures once before a transportless test attempt."""
asset_ids = image_asset_ids(definition)
if not asset_ids:
return {}
rows = (
await session.execute(select(TestAsset).where(TestAsset.id.in_(asset_ids)))
).scalars().all()
by_id = {row.id: row for row in rows}
missing = sorted(asset_ids - by_id.keys())
if missing:
raise ValueError("测试图片不存在: " + "、".join(missing))
pairs = await asyncio.gather(
*(
asyncio.to_thread(read_test_image, row.storage_key)
for row in rows
)
)
return {row.id: data for row, data in zip(rows, pairs, strict=True)}
backend/services/test_runs/evaluator.py
+2
View File
@@ -161,6 +161,7 @@ class EvaluationEngine:
id=expected_turn.id,
index=index,
user_input=expected_turn.user_input,
image=expected_turn.image,
assistant_reply="",
tool_calls=[],
runtime_records=[],
@@ -174,6 +175,7 @@ class EvaluationEngine:
id=raw.id,
index=index,
user_input=raw.user_input,
image=raw.image,
assistant_reply=raw.assistant_reply,
tool_calls=[item.as_result() for item in raw.tool_calls],
runtime_records=list(raw.runtime_records),
backend/services/test_runs/orchestrator.py
+20
View File
@@ -14,6 +14,7 @@ from db.session import SessionLocal
from models import RuntimeModelResource
from services.config_resolver import resolve_runtime_config
from services.runtime_variables import prepare_dynamic_config
from services.test_case_assets import load_test_asset_bytes
from services.test_runs.errors import TestExecutionError
from services.test_runs.evaluator import EvaluationEngine
from services.test_runs.text_runner import (
@@ -178,6 +179,19 @@ class TestRunOrchestrator:
snapshot = dict(row.case_snapshot or {})
definition = TestCaseDefinition.model_validate(snapshot)
max_attempts = row.max_attempts
try:
image_assets = await load_test_asset_bytes(session, definition)
except Exception as exc:
await self._mark_error(
run_case_id,
TestExecutionError(
code="TEST_IMAGE_READ_FAILED",
message=str(exc) or "测试图片无法读取",
stage="pipeline",
retryable=True,
),
)
return
for attempt in range(1, max_attempts + 1):
if control.stop_event.is_set():
@@ -212,6 +226,7 @@ class TestRunOrchestrator:
attempt_cfg,
definition,
timeout_seconds=timeout_seconds,
image_assets=image_assets,
on_turn_complete=save_turn_progress,
on_startup_complete=save_startup_progress,
)
@@ -346,6 +361,11 @@ class TestRunOrchestrator:
"id": turn.id,
"index": turn.index,
"userInput": turn.user_input,
"image": (
turn.image.model_dump(mode="json", by_alias=True)
if turn.image is not None
else None
),
"assistantReply": turn.assistant_reply,
"toolCalls": [item.as_result() for item in turn.tool_calls],
"runtimeRecords": list(turn.runtime_records),
backend/services/test_runs/text_runner.py
+83 -15
View File
@@ -55,7 +55,8 @@ from services.pipecat.service_factory import config_with_resource
from services.pipecat.workflow_services import build_workflow_llm_switcher
from services.test_runs.errors import TestExecutionError
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition
from services.vision import build_image_user_message
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef
IDLE_SETTLE_SECONDS = 0.2
@@ -104,6 +105,7 @@ class RawTurnResult:
user_input: str
assistant_reply: str
tool_calls: list[RawToolCall]
image: TestImageRef | None = None
runtime_records: list[dict[str, Any]] = field(default_factory=list)
@@ -359,6 +361,31 @@ def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
return messages
def _transcript_content(message: dict[str, Any]) -> str:
"""Keep evaluator transcripts readable without persisting image data URIs."""
content = message.get("content")
if isinstance(content, str):
return content.strip()
if not isinstance(content, list):
return ""
text_parts = [
str(part.get("text") or "").strip()
for part in content
if isinstance(part, dict)
and part.get("type") in {"text", "input_text"}
and str(part.get("text") or "").strip()
]
has_image = any(
isinstance(part, dict)
and part.get("type") in {"image_url", "input_image"}
for part in content
)
if has_image:
text_parts.append("[用户提交了一张测试图片]")
return "\n".join(text_parts).strip()
async def _wait_for_quiescence(
capture: _TextCaptureProcessor,
runner_task: asyncio.Task[None],
@@ -423,6 +450,7 @@ class TextPipelineRunner:
definition: TestCaseDefinition,
*,
timeout_seconds: int,
image_assets: dict[str, bytes] | None = None,
on_turn_complete: TurnCompleteCallback | None = None,
on_startup_complete: StartupCompleteCallback | None = None,
) -> TextCaseResult:
@@ -431,6 +459,7 @@ class TextPipelineRunner:
return await self._run(
cfg,
definition,
image_assets=image_assets or {},
on_turn_complete=on_turn_complete,
on_startup_complete=on_startup_complete,
)
@@ -447,6 +476,7 @@ class TextPipelineRunner:
cfg: AssistantConfig,
definition: TestCaseDefinition,
*,
image_assets: dict[str, bytes],
on_turn_complete: TurnCompleteCallback | None,
on_startup_complete: StartupCompleteCallback | None,
) -> TextCaseResult:
@@ -545,6 +575,7 @@ class TextPipelineRunner:
executor_holder: dict[str, MockToolExecutor] = {}
text_client_tools = TextClientToolPort()
workflow_vision_scope: dict[str, Any] = {"enabled": False}
def create_test_executor(store) -> MockToolExecutor:
executor = MockToolExecutor(store)
@@ -699,7 +730,7 @@ class TextPipelineRunner:
transport=None,
switch_services=switch_services,
set_knowledge_scope=knowledge.set_scope,
set_vision_scope=lambda _scope: None,
set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
vision_function=None,
set_input_enabled=lambda _enabled: None,
apply_turn_config=lambda _enabled, _config: asyncio.sleep(0),
@@ -809,26 +840,32 @@ class TextPipelineRunner:
retryable=False,
)
result = await self._run_turn(
cfg,
worker,
runner_task,
capture,
executor,
turn,
index,
image_assets=image_assets,
workflow_vision_scope=workflow_vision_scope,
)
results.append(result)
if on_turn_complete is not None:
await on_turn_complete(result)
transcript = [
{
"role": str(message.get("role") or ""),
"content": str(message.get("content") or ""),
}
for message in context.get_messages()
if message.get("role") in {"user", "assistant"}
and str(message.get("content") or "").strip()
]
transcript = []
for message in context.get_messages():
if message.get("role") not in {"user", "assistant"}:
continue
content = _transcript_content(message)
if content:
transcript.append(
{
"role": str(message.get("role") or ""),
"content": content,
}
)
return TextCaseResult(
turns=results,
transcript=transcript,
@@ -853,12 +890,16 @@ class TextPipelineRunner:
async def _run_turn(
self,
cfg: AssistantConfig,
worker: PipelineWorker,
runner_task: asyncio.Task[None],
capture: _TextCaptureProcessor,
executor: MockToolExecutor,
turn: FixedInputTurn,
index: int,
*,
image_assets: dict[str, bytes],
workflow_vision_scope: dict[str, Any],
) -> RawTurnResult:
tool_behaviors = [
behavior for behavior in turn.behaviors if behavior.type == "tool_call"
@@ -868,11 +909,37 @@ class TextPipelineRunner:
output_marker = len(capture.outputs)
tool_marker = len(capture.tool_calls)
runtime_record_marker = len(capture.runtime_records)
user_message: dict[str, Any] = {
"role": "user",
"content": turn.user_input,
}
if turn.image is not None:
image_bytes = image_assets.get(turn.image.asset_id)
if image_bytes is None:
raise TestExecutionError(
code="TEST_IMAGE_MISSING",
message=f"测试图片不存在: {turn.image.asset_id}",
stage="pipeline",
retryable=False,
)
try:
user_message = await build_image_user_message(
cfg,
image_bytes,
turn.user_input,
workflow_scope=(
workflow_vision_scope if cfg.type == "workflow" else None
),
)
except ValueError as exc:
raise TestExecutionError(
code="TEST_IMAGE_UNSUPPORTED",
message=str(exc),
stage="model",
retryable=False,
) from exc
await worker.queue_frame(
LLMMessagesAppendFrame(
messages=[{"role": "user", "content": turn.user_input}],
run_llm=True,
)
LLMMessagesAppendFrame(messages=[user_message], run_llm=True)
)
await _wait_for_quiescence(capture, runner_task, activity_marker)
if executor.unmocked_calls:
@@ -886,6 +953,7 @@ class TextPipelineRunner:
id=turn.id,
index=index,
user_input=turn.user_input,
image=turn.image,
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
tool_calls=list(capture.tool_calls[tool_marker:]),
runtime_records=list(
backend/services/vision.py
+89
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import asyncio
import base64
from dataclasses import dataclass
from io import BytesIO
from typing import Any
@@ -30,6 +31,14 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
)
@dataclass(frozen=True)
class VisionInputResolution:
"""The active model path for one explicit user image turn."""
native: bool
config: AssistantConfig
def _require(value: str, label: str) -> str:
if value:
return value
@@ -246,3 +255,83 @@ def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
result.vision_llm_api_key = cfg.llm_api_key
result.vision_llm_base_url = cfg.llm_base_url
return result
def resolve_vision_input(
cfg: AssistantConfig,
workflow_scope: dict[str, Any] | None = None,
) -> VisionInputResolution:
"""Resolve native versus auxiliary vision using the active runtime scope."""
if cfg.type != "workflow":
if not cfg.vision_enabled:
raise ValueError("当前助手未开启视觉能力")
if cfg.vision_model_resource_id:
if not cfg.vision_llm_support_image_input:
raise ValueError("当前视觉模型不支持图片输入")
return VisionInputResolution(native=False, config=cfg)
if not cfg.llm_support_image_input:
raise ValueError("当前大语言模型不支持图片输入")
return VisionInputResolution(native=True, config=cfg)
scope = workflow_scope or {}
if not scope.get("enabled"):
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
vision_resource_id = str(scope.get("vision_model_resource_id") or "")
llm_resource_id = str(scope.get("llm_resource_id") or "")
resource_id = vision_resource_id or llm_resource_id
resource = cfg.workflow_model_resources.get(resource_id)
if resource is not None:
if not resource.support_image_input:
raise ValueError(f"当前视觉模型不支持图片输入: {resource.id}")
result = config_with_vision_resource(cfg, resource)
elif not vision_resource_id:
result = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载: {vision_resource_id}")
result.vision_prompt_mode = normalize_vision_prompt_mode(
scope.get("vision_prompt_mode")
)
result.vision_prompt = str(scope.get("vision_prompt") or "").strip()
return VisionInputResolution(
native=not bool(vision_resource_id),
config=result,
)
async def build_image_user_message(
cfg: AssistantConfig,
image_bytes: bytes,
prompt_text: str,
*,
workflow_scope: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Build the same model-facing user message used by an explicit photo turn."""
question = prompt_text.strip() or "请根据用户刚提交的图片进行回复。"
frame = await asyncio.to_thread(image_frame_from_jpeg, image_bytes)
resolution = resolve_vision_input(cfg, workflow_scope)
if resolution.native:
data_uri = await asyncio.to_thread(image_data_uri, frame)
return {
"role": "user",
"content": [
{"type": "text", "text": question},
{"type": "image_url", "image_url": {"url": data_uri}},
],
}
observation = await analyze_image_with_vision_model(
resolution.config,
frame,
question,
)
return {
"role": "user",
"content": (
f"{question}\n\n"
"[视觉模型对用户刚提交图片的观察]\n"
f"{observation or '视觉模型没有返回有效观察结果。'}"
),
}
backend/test_schemas.py
+25 -8
View File
@@ -143,20 +143,36 @@ ExpectedBehavior = Annotated[
]
class FixedInputTurn(CamelModel):
id: str = Field(min_length=1, max_length=128)
user_input: str = Field(min_length=1, max_length=20_000)
behaviors: list[ExpectedBehavior] = Field(default_factory=list, max_length=100)
class TestImageRef(CamelModel):
asset_id: str = Field(min_length=1, max_length=40)
filename: str = Field(default="image.jpg", max_length=255)
width: int = Field(ge=1)
height: int = Field(ge=1)
size_bytes: int = Field(default=0, ge=0)
@field_validator("user_input")
@field_validator("asset_id", "filename")
@classmethod
def strip_user_input(cls, value: str) -> str:
def strip_image_text(cls, value: str) -> str:
value = value.strip()
if not value:
raise ValueError("用户输入不能为空")
raise ValueError("图片资源信息不完整")
return value
class FixedInputTurn(CamelModel):
id: str = Field(min_length=1, max_length=128)
user_input: str = Field(default="", max_length=20_000)
image: TestImageRef | None = None
behaviors: list[ExpectedBehavior] = Field(default_factory=list, max_length=100)
@model_validator(mode="after")
def validate_input(self):
self.user_input = self.user_input.strip()
if not self.user_input and self.image is None:
raise ValueError("用户输入和图片至少填写一项")
return self
class OverallCriterion(CamelModel):
id: str = Field(min_length=1, max_length=128)
type: Literal["llm"] = "llm"
@@ -272,7 +288,7 @@ class TestCaseWrite(CamelModel):
@model_validator(mode="after")
def validate_mvp_definition(self):
if self.input_mode != "fixed_script_text":
raise ValueError("第一版只支持固定脚本 · 文字")
raise ValueError("第一版只支持固定脚本 · 多模态")
TestCaseDefinition(
context_turns=self.context_turns,
turns=self.turns,
@@ -362,6 +378,7 @@ class BatchTurnResult(CamelModel):
id: str
index: int
user_input: str
image: TestImageRef | None = None
assistant_reply: str
tool_calls: list[BatchToolCallRecord] = Field(default_factory=list)
runtime_records: list[dict[str, Any]] = Field(default_factory=list)
backend/tests/test_pipeline_knowledge.py
+14 -7
View File
@@ -9,8 +9,8 @@ from services.pipecat.pipeline import (
KnowledgeRetrievalProcessor,
UserTurnRoutingProcessor,
_knowledge_tool_description,
_workflow_vision_uses_main_llm,
)
from services.vision import resolve_vision_input
class KnowledgeToolDescriptionTest(unittest.TestCase):
@@ -159,29 +159,36 @@ class WorkflowVisionModeTest(unittest.TestCase):
capability="LLM",
interface_type="openai-llm",
support_image_input=True,
)
),
"auxiliary_vision": RuntimeModelResource(
id="auxiliary_vision",
name="独立视觉模型",
capability="LLM",
interface_type="openai-llm",
support_image_input=True,
),
},
)
self.assertTrue(
_workflow_vision_uses_main_llm(
resolve_vision_input(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": None,
},
)
).native
)
self.assertFalse(
_workflow_vision_uses_main_llm(
resolve_vision_input(
cfg,
{
"enabled": True,
"llm_resource_id": "agent_llm",
"vision_model_resource_id": "auxiliary_vision",
},
)
).native
)
def test_rejects_a_non_visual_active_agent_llm(self):
@@ -199,7 +206,7 @@ class WorkflowVisionModeTest(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "不支持图片输入"):
_workflow_vision_uses_main_llm(
resolve_vision_input(
cfg,
{
"enabled": True,
backend/tests/test_test_image_inputs.py
+118
View File
@@ -0,0 +1,118 @@
from io import BytesIO
import unittest
from unittest.mock import AsyncMock, patch
from PIL import Image
from pydantic import ValidationError
from models import AssistantConfig
from services.test_assets import read_test_image, store_test_image
from services.test_runs.text_runner import _transcript_content
from services.vision import build_image_user_message
from test_schemas import FixedInputTurn, TestImageRef
def _jpeg_bytes() -> bytes:
buffer = BytesIO()
Image.new("RGB", (12, 8), (220, 40, 40)).save(buffer, format="JPEG")
return buffer.getvalue()
def _image_ref() -> TestImageRef:
return TestImageRef(
asset_id="ta_fixture",
filename="fixture.jpg",
width=12,
height=8,
size_bytes=128,
)
class TestImageSchemaTests(unittest.TestCase):
def test_allows_image_only_turn(self):
turn = FixedInputTurn(id="turn_1", image=_image_ref())
self.assertEqual(turn.user_input, "")
self.assertEqual(turn.image.asset_id, "ta_fixture")
def test_rejects_turn_without_text_or_image(self):
with self.assertRaisesRegex(ValidationError, "至少填写一项"):
FixedInputTurn(id="turn_1")
class PersistentTestAssetTests(unittest.TestCase):
def test_normalizes_and_stores_under_test_prefix(self):
with patch("services.test_assets.put_object") as put_object:
stored = store_test_image("ta_fixture", _jpeg_bytes())
self.assertEqual(stored.storage_key, "test-assets/ta_fixture.jpg")
self.assertEqual((stored.width, stored.height), (12, 8))
self.assertEqual(put_object.call_args.args[0], stored.storage_key)
self.assertEqual(put_object.call_args.args[2], "image/jpeg")
def test_reads_only_test_asset_keys(self):
with patch("services.test_assets.get_object", return_value=b"jpeg"):
self.assertEqual(read_test_image("test-assets/ta_fixture.jpg"), b"jpeg")
with self.assertRaisesRegex(ValueError, "存储位置无效"):
read_test_image("conversation-inputs/unsafe.jpg")
class ImageUserMessageTests(unittest.IsolatedAsyncioTestCase):
async def test_native_model_receives_multimodal_message(self):
message = await build_image_user_message(
AssistantConfig(
type="prompt",
vision_enabled=True,
llm_support_image_input=True,
),
_jpeg_bytes(),
"这是什么颜色？",
)
self.assertEqual(message["role"], "user")
self.assertEqual(message["content"][0]["text"], "这是什么颜色？")
self.assertTrue(
message["content"][1]["image_url"]["url"].startswith(
"data:image/jpeg;base64,"
)
)
async def test_auxiliary_model_observation_becomes_text_context(self):
with patch(
"services.vision.analyze_image_with_vision_model",
new=AsyncMock(return_value="画面中是一个红色方块。"),
):
message = await build_image_user_message(
AssistantConfig(
type="prompt",
vision_enabled=True,
vision_model_resource_id="vision_model",
vision_llm_support_image_input=True,
),
_jpeg_bytes(),
"画面里有什么？",
)
self.assertIsInstance(message["content"], str)
self.assertIn("红色方块", message["content"])
def test_transcript_does_not_persist_image_data(self):
content = _transcript_content(
{
"role": "user",
"content": [
{"type": "text", "text": "请看图"},
{
"type": "image_url",
"image_url": {"url": "data:image/jpeg;base64,secret"},
},
],
}
)
self.assertEqual(content, "请看图\n[用户提交了一张测试图片]")
self.assertNotIn("base64", content)
if __name__ == "__main__":
unittest.main()
frontend/src/components/assistant-editor/debug-auto-test.tsx
+12 -1
View File
@@ -51,6 +51,7 @@ import {
import {
batchRunsApi,
modelResourcesApi,
testAssetsApi,
testCasesApi,
testSuitesApi,
type ModelResource,
@@ -101,7 +102,7 @@ function isRunFinished(run: BatchRunSnapshot): boolean {
function runnableMessage(item: TestCase): string {
if (item.inputMode !== "fixed_script_text") {
return "当前自动测试仅支持固定脚本 · 文字";
return "当前自动测试仅支持固定脚本 · 多模态";
}
return getTestCaseValidationMessage(item) ?? "";
}
@@ -653,6 +654,16 @@ function AutoTestTimeline({
role: "user",
label: `固定输入 · ${turn.index + 1} / ${maxTurns}`,
content: turn.userInput,
attachments: turn.image
? [
{
id: turn.image.assetId,
type: "image",
url: testAssetsApi.contentUrl(turn.image.assetId),
alt: turn.image.filename,
},
]
: undefined,
},
});
items.push({
frontend/src/components/batch-test/batch-case-detail-drawer.tsx
+15 -4
View File
@@ -28,6 +28,7 @@ import {
type BatchToolCallRecord,
type BatchTurnResult,
} from "@/data/batch-run";
import { testAssetsApi } from "@/lib/api";
import { cn } from "@/lib/utils";
type BatchCaseDetailDrawerProps = {
@@ -98,7 +99,7 @@ export function BatchCaseDetailDrawer({
<section className="rounded-2xl border border-hairline bg-canvas-soft px-4 py-3.5">
<p className="text-xs font-medium text-foreground">执行契约</p>
<p className="mt-1 text-xs leading-5 text-muted-soft">
正在执行固定文字脚本；完成后将展示每一轮回复、工具调用和全部评估标准。
正在执行固定多模态脚本；完成后将展示每轮回复、工具调用和评估标准。
</p>
</section>
<ExecutionTimeline status={item.status} />
@@ -235,9 +236,19 @@ function TurnResultCard({ turn }: { turn: BatchTurnResult }) {
</p>
<div className="mt-2 grid gap-2 text-xs leading-5 sm:grid-cols-[72px_minmax(0,1fr)]">
<span className="text-muted-soft">用户输入</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.userInput}
</p>
<div className="space-y-2 text-muted-foreground">
{turn.image ? (
// eslint-disable-next-line @next/next/no-img-element
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="max-h-48 w-full max-w-xs rounded-xl object-cover"
/>
) : null}
{turn.userInput ? (
<p className="whitespace-pre-wrap">{turn.userInput}</p>
) : null}
</div>
<span className="text-muted-soft">Agent 回复</span>
<p className="whitespace-pre-wrap text-muted-foreground">
{turn.assistantReply}
frontend/src/components/pages/TestCasesPage.tsx
+5 -1
View File
@@ -112,6 +112,8 @@ function SuiteListView() {
}
useEffect(() => {
// Initial API hydration is intentionally owned by this mount effect.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reloadSuites();
}, []);
@@ -507,6 +509,8 @@ function SuiteDetailView({
}
useEffect(() => {
// Suite changes remount this view; hydrate the selected case once.
// eslint-disable-next-line react-hooks/set-state-in-effect
void reload(initialCaseId ?? null);
// suiteId changes by remount; the initial request only runs once per view.
// eslint-disable-next-line react-hooks/exhaustive-deps
@@ -931,7 +935,7 @@ function SuiteDetailView({
{TEST_CASE_INPUT_MODE_LABEL[draft.inputMode]} · 开发中
</div>
<p className="max-w-sm text-xs leading-5 text-muted-foreground">
当前仅支持「固定脚本 · 文字」编辑器，其它输入模式稍后接入。
当前仅支持「固定脚本 · 多模态」编辑器，其它输入模式稍后接入。
</p>
</div>
)}
frontend/src/components/test-cases/input-mode-picker.tsx
+2 -2
View File
@@ -37,8 +37,8 @@ const FIXED_SCRIPT_OPTIONS: {
}[] = [
{
value: "fixed_script_text",
title: "文字",
description: "使用固定文字逐轮测试",
title: "文字与图片",
description: "使用固定文字或图片逐轮测试",
icon: Type,
available: true,
},
frontend/src/components/test-cases/next-reply-editor.tsx
+131 -4
View File
@@ -2,7 +2,7 @@
/**
* 测试用例编辑器主体：
* 对话上下文 → 固定文字脚本轮次 → 整体评估标准。
* 对话上下文 → 固定多模态脚本轮次 → 整体评估标准。
*/
import {
@@ -26,6 +26,8 @@ import {
ChevronDown,
ChevronUp,
GripVertical,
ImagePlus,
Loader2,
MessageSquareText,
MessagesSquare,
Plus,
@@ -71,12 +73,13 @@ import {
type KeywordMatchMode,
type OverallCriterion,
type ReplyExpectedBehavior,
type TestImageRef,
type TestCaseInputMode,
type ToolCallExpectedBehavior,
type ToolParamAssertion,
type ToolParamMatchMode,
} from "@/data/test-suites";
import { toolsApi, type Tool } from "@/lib/api";
import { testAssetsApi, toolsApi, type Tool } from "@/lib/api";
import { cn } from "@/lib/utils";
export type CaseEditorDraft = {
@@ -637,6 +640,9 @@ export function NextReplyEditorBody({
onChangeUserInput={(userInput) =>
updateTurn(turn.id, { userInput })
}
onChangeImage={(image) =>
updateTurn(turn.id, { image })
}
onAddReply={() =>
addBehavior(turn.id, createReplyBehavior("llm"))
}
@@ -812,6 +818,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds,
onToggleBehavior,
onChangeUserInput,
onChangeImage,
onAddReply,
onAddToolCall,
onUpdateBehavior,
@@ -825,6 +832,7 @@ function SortableFixedInputTurn({
expandedBehaviorIds: Set<string>;
onToggleBehavior: (behaviorId: string) => void;
onChangeUserInput: (value: string) => void;
onChangeImage: (value: TestImageRef | null) => void;
onAddReply: () => void;
onAddToolCall: () => void;
onUpdateBehavior: (behavior: ExpectedBehavior) => void;
@@ -840,6 +848,34 @@ function SortableFixedInputTurn({
isDragging,
} = useSortable({ id: turn.id });
const [addOpen, setAddOpen] = useState(false);
const fileInputRef = useRef<HTMLInputElement>(null);
const [uploadingImage, setUploadingImage] = useState(false);
const [imageError, setImageError] = useState("");
async function uploadImage(file: File | undefined) {
if (!file || uploadingImage) return;
if (!file.type.startsWith("image/")) {
setImageError("请选择图片文件");
return;
}
setUploadingImage(true);
setImageError("");
try {
const asset = await testAssetsApi.uploadImage(file);
onChangeImage({
assetId: asset.id,
filename: asset.filename,
width: asset.width,
height: asset.height,
sizeBytes: asset.sizeBytes,
});
} catch (error) {
setImageError(error instanceof Error ? error.message : "图片上传失败");
} finally {
setUploadingImage(false);
if (fileInputRef.current) fileInputRef.current.value = "";
}
}
const style = {
transform: CSS.Transform.toString(transform),
@@ -888,17 +924,108 @@ function SortableFixedInputTurn({
</div>
<div className="space-y-3 pl-1">
<div className="space-y-1.5">
<div
className="space-y-1.5"
onDragOver={(event) => {
if (event.dataTransfer.types.includes("Files")) event.preventDefault();
}}
onDrop={(event) => {
const file = Array.from(event.dataTransfer.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
>
<div className="text-xs font-medium text-muted-foreground">
用户输入
</div>
<Textarea
value={turn.userInput}
onChange={(event) => onChangeUserInput(event.target.value)}
placeholder="输入本轮用户原话…"
onPaste={(event) => {
const file = Array.from(event.clipboardData.files).find((item) =>
item.type.startsWith("image/"),
);
if (!file) return;
event.preventDefault();
void uploadImage(file);
}}
placeholder={
turn.image
? "可选：补充用户与图片一起提交的问题…"
: "输入本轮用户原话，或添加一张图片…"
}
rows={2}
className="field-sizing-fixed min-h-[56px] resize-y border-hairline-strong bg-background text-sm"
/>
{turn.image ? (
<div className="flex items-center gap-3 rounded-xl border border-hairline bg-canvas-soft p-2">
{/* eslint-disable-next-line @next/next/no-img-element */}
<img
src={testAssetsApi.contentUrl(turn.image.assetId)}
alt={turn.image.filename}
className="h-16 w-20 shrink-0 rounded-lg object-cover"
/>
<div className="min-w-0 flex-1">
<div className="truncate text-xs font-medium text-foreground">
{turn.image.filename}
</div>
<div className="mt-1 text-[11px] text-muted-soft">
{turn.image.width} × {turn.image.height}
{turn.image.sizeBytes > 0
? ` · ${Math.max(1, Math.round(turn.image.sizeBytes / 1024))} KB`
: ""}
</div>
</div>
<Button
type="button"
variant="ghost"
size="icon-sm"
className="shrink-0 text-muted-soft hover:text-destructive"
onClick={() => {
setImageError("");
onChangeImage(null);
}}
aria-label="移除本轮图片"
>
<Trash2 size={14} />
</Button>
</div>
) : null}
<div className="flex flex-wrap items-center gap-2">
<input
ref={fileInputRef}
type="file"
accept="image/*"
className="sr-only"
onChange={(event) => void uploadImage(event.target.files?.[0])}
/>
<Button
type="button"
variant="outline"
size="sm"
className="h-8 gap-1.5 rounded-full border-hairline-strong text-xs text-muted-foreground"
disabled={uploadingImage}
onClick={() => fileInputRef.current?.click()}
>
{uploadingImage ? (
<Loader2 size={13} className="animate-spin" />
) : (
<ImagePlus size={13} />
)}
{turn.image ? "替换图片" : "添加图片"}
</Button>
<span className="text-[11px] text-muted-soft">
支持选择、粘贴或拖入；每轮最多一张
</span>
</div>
{imageError ? (
<p className="text-[11px] text-destructive" role="alert">
{imageError}
</p>
) : null}
</div>
<div className="space-y-2">
frontend/src/data/batch-run.ts
+7
View File
@@ -54,6 +54,13 @@ export type BatchTurnResult = {
id: string;
index: number;
userInput: string;
image: {
assetId: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
} | null;
assistantReply: string;
toolCalls: BatchToolCallRecord[];
runtimeRecords?: Array<Record<string, unknown>>;
frontend/src/data/test-suites.ts
+17 -6
View File
@@ -1,7 +1,7 @@
/**
* 测试集 / 测试用例的前后端共享契约与编辑器辅助函数。
* 两层：Test Suite → Test Case。
* MVP 仅运行固定文字脚本；其它输入模式保留为产品路线提示。
* MVP 运行固定文字/图片脚本；其它输入模式保留为产品路线提示。
*/
/** 顶部模式选择器的唯一来源；不再单独维护「测试类型」字段 */
@@ -80,6 +80,14 @@ export type ToolCallExpectedBehavior = {
export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
export type TestImageRef = {
assetId: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
};
/**
* 固定输入的一轮：只编辑 User 输入；Agent 回复 / Tool Call 由运行时生成。
* behaviors 为 0~N 个预期行为（AND）；空数组表示仅推动对话、不做断言。
@@ -87,6 +95,7 @@ export type ExpectedBehavior = ReplyExpectedBehavior | ToolCallExpectedBehavior;
export type FixedInputTurn = {
id: string;
userInput: string;
image: TestImageRef | null;
behaviors: ExpectedBehavior[];
};
@@ -134,7 +143,7 @@ export const DEFAULT_INPUT_MODE: TestCaseInputMode = "fixed_script_text";
/**
* MVP 可运行性规则。
* 目前执行器只支持固定文字；每一轮都需要用户输入，并至少配置一项断言。
* 目前执行器支持固定文字和单张图片；每轮至少有一种输入，并配置断言。
*/
export function getTestCaseValidationMessage(
item: Pick<
@@ -219,10 +228,10 @@ export function getTestCaseValidationMessage(
.join("、")}`;
}
const emptyTurnIndex = item.turns.findIndex(
(turn) => !turn.userInput.trim(),
(turn) => !turn.userInput.trim() && !turn.image,
);
if (emptyTurnIndex >= 0) {
return `第 ${emptyTurnIndex + 1} 轮用户输入不能为空`;
return `第 ${emptyTurnIndex + 1} 轮需要填写用户文字或添加图片`;
}
for (let turnIndex = 0; turnIndex < item.turns.length; turnIndex += 1) {
const turn = item.turns[turnIndex];
@@ -267,7 +276,7 @@ export function isTestCaseRunnable(
}
export const TEST_CASE_INPUT_MODE_LABEL: Record<TestCaseInputMode, string> = {
fixed_script_text: "固定脚本 · 文字",
fixed_script_text: "固定脚本 · 多模态",
fixed_script_turn_voice: "固定脚本 · 逐轮语音",
fixed_script_continuous_voice: "固定脚本 · 连续语音",
user_sim_text: "用户模拟 · 文字",
@@ -279,7 +288,7 @@ export const TEST_CASE_INPUT_MODE_SHORT_LABEL: Record<
TestCaseInputMode,
string
> = {
fixed_script_text: "固定文字",
fixed_script_text: "固定输入",
fixed_script_turn_voice: "逐轮语音",
fixed_script_continuous_voice: "连续语音",
user_sim_text: "模拟文字",
@@ -412,6 +421,7 @@ export function createEmptyFixedInputTurn(
return {
id: newEditorId("turn"),
userInput,
image: null,
behaviors: [],
};
}
@@ -484,6 +494,7 @@ export function cloneTurns(turns: FixedInputTurn[]): FixedInputTurn[] {
return turns.map((turn) => ({
id: turn.id,
userInput: turn.userInput,
image: turn.image ? { ...turn.image } : null,
behaviors: turn.behaviors.map(cloneBehavior),
}));
}
frontend/src/lib/api.ts
+27
View File
@@ -854,6 +854,33 @@ export type TestCaseWrite = {
overallCriteria: OverallCriterion[];
};
export type TestImageAsset = {
id: string;
filename: string;
width: number;
height: number;
sizeBytes: number;
contentUrl: string;
};
export const testAssetsApi = {
uploadImage: (file: File) => {
const body = new FormData();
body.append("file", file);
return request<TestImageAsset>("/api/test-assets/images", {
method: "POST",
body,
});
},
contentUrl: (assetId: string) =>
`${API_BASE}/api/test-assets/${encodeURIComponent(assetId)}/content`,
remove: (assetId: string) =>
request<{ ok: boolean }>(
`/api/test-assets/${encodeURIComponent(assetId)}`,
{ method: "DELETE" },
),
};
export const testSuitesApi = {
list: () => request<TestSuite[]>("/api/test-suites"),
get: (id: string) => request<TestSuite>(`/api/test-suites/${id}`),