From 9e4dfbe596e0d932ed4e427c9b59a2e6d8219787 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Sun, 23 Aug 2026 23:46:18 +0800 Subject: [PATCH] feat: support image inputs in assistant tests --- backend/app.py | 2 + backend/db/models.py | 17 ++ .../versions/20260818_0023_add_test_assets.py | 53 ++++++ backend/routes/test_assets.py | 172 ++++++++++++++++++ backend/routes/test_cases.py | 25 ++- backend/routes/test_runs.py | 19 +- backend/services/image_processing.py | 41 +++++ backend/services/input_assets.py | 36 +--- backend/services/pipecat/pipeline.py | 60 +----- backend/services/test_assets.py | 44 +++++ backend/services/test_case_assets.py | 72 ++++++++ backend/services/test_runs/evaluator.py | 2 + backend/services/test_runs/orchestrator.py | 20 ++ backend/services/test_runs/text_runner.py | 98 ++++++++-- backend/services/vision.py | 89 +++++++++ backend/test_schemas.py | 33 +++- backend/tests/test_pipeline_knowledge.py | 21 ++- backend/tests/test_test_image_inputs.py | 118 ++++++++++++ .../assistant-editor/debug-auto-test.tsx | 13 +- .../batch-test/batch-case-detail-drawer.tsx | 19 +- .../src/components/pages/TestCasesPage.tsx | 6 +- .../test-cases/input-mode-picker.tsx | 4 +- .../test-cases/next-reply-editor.tsx | 135 +++++++++++++- frontend/src/data/batch-run.ts | 7 + frontend/src/data/test-suites.ts | 23 ++- frontend/src/lib/api.ts | 27 +++ 26 files changed, 1010 insertions(+), 146 deletions(-) create mode 100644 backend/migrations/versions/20260818_0023_add_test_assets.py create mode 100644 backend/routes/test_assets.py create mode 100644 backend/services/image_processing.py create mode 100644 backend/services/test_assets.py create mode 100644 backend/services/test_case_assets.py create mode 100644 backend/tests/test_test_image_inputs.py diff --git a/backend/app.py b/backend/app.py index 50789f0..2ddfa66 100644 --- a/backend/app.py +++ b/backend/app.py @@ -48,6 +48,7 @@ from routes import ( realtime_api_keys, site_settings, test_cases, + test_assets, test_runs, tools, voice_webrtc, @@ -104,6 +105,7 @@ app.include_router(openai_realtime.router) app.include_router(realtime_api_keys.router) app.include_router(site_settings.router) app.include_router(test_cases.router) +app.include_router(test_assets.router) app.include_router(test_runs.router) app.include_router(tools.router) app.include_router(voice_webrtc.router) diff --git a/backend/db/models.py b/backend/db/models.py index 478b0aa..f83313d 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -475,6 +475,23 @@ class TestSuite(Base): ) +class TestAsset(Base): + """Immutable image fixture referenced by one or more test definitions.""" + + __tablename__ = "test_assets" + + id: Mapped[str] = mapped_column(String(40), primary_key=True) + original_name: Mapped[str] = mapped_column(String(255), default="image.jpg") + storage_key: Mapped[str] = mapped_column(String(1024), unique=True) + mime_type: Mapped[str] = mapped_column(String(128), default="image/jpeg") + width: Mapped[int] = mapped_column(Integer) + height: Mapped[int] = mapped_column(Integer) + size_bytes: Mapped[int] = mapped_column(Integer) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + class TestCase(Base): """One editable test definition; nested turns remain one JSON document.""" diff --git a/backend/migrations/versions/20260818_0023_add_test_assets.py b/backend/migrations/versions/20260818_0023_add_test_assets.py new file mode 100644 index 0000000..11a4072 --- /dev/null +++ b/backend/migrations/versions/20260818_0023_add_test_assets.py @@ -0,0 +1,53 @@ +"""add persistent image fixtures for test cases + +Revision ID: 20260818_0023 +Revises: 20260813_0022 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260818_0023" +down_revision: str | Sequence[str] | None = "20260813_0022" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "test_assets", + sa.Column("id", sa.String(length=40), nullable=False), + sa.Column( + "original_name", + sa.String(length=255), + server_default="image.jpg", + nullable=False, + ), + sa.Column("storage_key", sa.String(length=1024), nullable=False), + sa.Column( + "mime_type", + sa.String(length=128), + server_default="image/jpeg", + nullable=False, + ), + sa.Column("width", sa.Integer(), nullable=False), + sa.Column("height", sa.Integer(), nullable=False), + sa.Column("size_bytes", sa.Integer(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("storage_key"), + ) + + +def downgrade() -> None: + op.drop_table("test_assets") diff --git a/backend/routes/test_assets.py b/backend/routes/test_assets.py new file mode 100644 index 0000000..4093aea --- /dev/null +++ b/backend/routes/test_assets.py @@ -0,0 +1,172 @@ +"""Persistent image fixtures for repeatable assistant tests.""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from uuid import uuid4 + +from db.models import TestAsset, TestCase, TestRunCase +from db.session import get_session +from fastapi import APIRouter, Depends, File, HTTPException, UploadFile +from fastapi.responses import Response +from loguru import logger +from pydantic import BaseModel +from services.auth import require_admin +from services.test_assets import delete_test_image, read_test_image, store_test_image +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +import settings + + +router = APIRouter( + prefix="/api/test-assets", + tags=["test-assets"], + dependencies=[Depends(require_admin)], +) + + +class TestImageAssetOut(BaseModel): + id: str + filename: str + width: int + height: int + sizeBytes: int + contentUrl: str + + +def _new_id() -> str: + return f"ta_{uuid4().hex}" + + +def _safe_filename(value: str | None) -> str: + name = Path(value or "image.jpg").name.strip() + return (name or "image.jpg")[:255] + + +def _out(row: TestAsset) -> TestImageAssetOut: + return TestImageAssetOut( + id=row.id, + filename=row.original_name, + width=row.width, + height=row.height, + sizeBytes=row.size_bytes, + contentUrl=f"/api/test-assets/{row.id}/content", + ) + + +def _contains_asset_id(value: object, asset_id: str) -> bool: + if isinstance(value, dict): + if value.get("assetId") == asset_id or value.get("asset_id") == asset_id: + return True + return any(_contains_asset_id(item, asset_id) for item in value.values()) + if isinstance(value, list): + return any(_contains_asset_id(item, asset_id) for item in value) + return False + + +async def _is_referenced(session: AsyncSession, asset_id: str) -> bool: + definitions = ( + await session.execute(select(TestCase.definition)) + ).scalars().all() + if any(_contains_asset_id(value, asset_id) for value in definitions): + return True + snapshots = ( + await session.execute(select(TestRunCase.case_snapshot)) + ).scalars().all() + return any(_contains_asset_id(value, asset_id) for value in snapshots) + + +@router.post("/images", response_model=TestImageAssetOut) +async def upload_test_image( + file: UploadFile = File(...), + session: AsyncSession = Depends(get_session), +): + data = await file.read(settings.INPUT_IMAGE_MAX_BYTES + 1) + if len(data) > settings.INPUT_IMAGE_MAX_BYTES: + limit_mb = settings.INPUT_IMAGE_MAX_BYTES // 1024 // 1024 + raise HTTPException(413, f"图片不能超过 {limit_mb} MB") + if not data: + raise HTTPException(400, "图片文件不能为空") + + asset_id = _new_id() + try: + stored = await asyncio.to_thread(store_test_image, asset_id, data) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + except Exception as exc: + raise HTTPException(503, "图片暂时无法上传，请确认对象存储可用") from exc + + row = TestAsset( + id=asset_id, + original_name=_safe_filename(file.filename), + storage_key=stored.storage_key, + mime_type=stored.mime_type, + width=stored.width, + height=stored.height, + size_bytes=stored.size_bytes, + ) + session.add(row) + try: + await session.commit() + except Exception: + await session.rollback() + try: + await asyncio.to_thread(delete_test_image, stored.storage_key) + except Exception: + logger.warning(f"回滚测试图片上传时清理对象失败: {stored.storage_key}") + raise + await session.refresh(row) + return _out(row) + + +@router.get("/{asset_id}", response_model=TestImageAssetOut) +async def get_test_asset( + asset_id: str, + session: AsyncSession = Depends(get_session), +): + row = await session.get(TestAsset, asset_id) + if row is None: + raise HTTPException(404, "测试图片不存在") + return _out(row) + + +@router.get("/{asset_id}/content") +async def get_test_asset_content( + asset_id: str, + session: AsyncSession = Depends(get_session), +): + row = await session.get(TestAsset, asset_id) + if row is None: + raise HTTPException(404, "测试图片不存在") + try: + data = await asyncio.to_thread(read_test_image, row.storage_key) + except Exception as exc: + raise HTTPException(503, "测试图片暂时无法读取") from exc + return Response( + content=data, + media_type=row.mime_type, + headers={"Cache-Control": "private, max-age=3600"}, + ) + + +@router.delete("/{asset_id}") +async def delete_test_asset( + asset_id: str, + session: AsyncSession = Depends(get_session), +): + row = await session.get(TestAsset, asset_id) + if row is None: + raise HTTPException(404, "测试图片不存在") + if await _is_referenced(session, asset_id): + raise HTTPException(409, "测试图片仍被用例或运行记录引用") + + storage_key = row.storage_key + await session.delete(row) + await session.commit() + try: + await asyncio.to_thread(delete_test_image, storage_key) + except Exception: + logger.warning(f"测试图片数据已删除，但对象存储清理失败: {storage_key}") + return {"ok": True} diff --git a/backend/routes/test_cases.py b/backend/routes/test_cases.py index 1a48b1c..b957101 100644 --- a/backend/routes/test_cases.py +++ b/backend/routes/test_cases.py @@ -9,6 +9,7 @@ from db.models import TestCase, TestRunCase, TestSuite from db.session import get_session from fastapi import APIRouter, Depends, HTTPException, Query from services.auth import require_admin +from services.test_case_assets import canonicalize_test_assets from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from test_schemas import ( @@ -79,13 +80,6 @@ def _case_out(row: TestCase) -> TestCaseOut: ) -def _apply_case_write(row: TestCase, body: TestCaseWrite) -> None: - row.name = body.name.strip() - row.description = body.description.strip() - row.input_mode = body.input_mode - row.definition = body.definition().model_dump(mode="json", by_alias=True) - - async def _renumber_cases(session: AsyncSession, suite_id: str) -> None: rows = ( await session.execute( @@ -232,13 +226,18 @@ async def create_test_case( max_order = await session.scalar( select(func.max(TestCase.sort_order)).where(TestCase.suite_id == suite_id) ) + definition = body.definition() + try: + await canonicalize_test_assets(session, definition) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc row = TestCase( id=_new_id("tc"), suite_id=suite_id, name=body.name.strip(), description=body.description.strip(), input_mode=body.input_mode, - definition=body.definition().model_dump(mode="json", by_alias=True), + definition=definition.model_dump(mode="json", by_alias=True), sort_order=int(max_order if max_order is not None else -1) + 1, last_result="not_run", ) @@ -315,7 +314,15 @@ async def update_test_case( row = await session.get(TestCase, case_id) if row is None: raise HTTPException(404, "测试用例不存在") - _apply_case_write(row, body) + definition = body.definition() + try: + await canonicalize_test_assets(session, definition) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + row.name = body.name.strip() + row.description = body.description.strip() + row.input_mode = body.input_mode + row.definition = definition.model_dump(mode="json", by_alias=True) await _touch_suite(session, row.suite_id) await session.commit() await session.refresh(row) diff --git a/backend/routes/test_runs.py b/backend/routes/test_runs.py index 80a3eea..64caa3b 100644 --- a/backend/routes/test_runs.py +++ b/backend/routes/test_runs.py @@ -18,6 +18,8 @@ from fastapi import APIRouter, Depends, HTTPException from services.auth import require_admin from services.config_resolver import resolve_runtime_config from services.runtime_variables import DynamicVariableError, prepare_dynamic_config +from services.realtime.launcher import validate_visual_runtime +from services.test_case_assets import canonicalize_test_assets, image_asset_ids from services.test_runs.orchestrator import test_run_orchestrator from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -140,13 +142,18 @@ async def create_test_run( raise HTTPException(422, "测试用例不存在: " + "、".join(missing)) ordered_cases = [by_id[case_id] for case_id in body.case_ids] tools_by_id = {tool.id: tool for tool in runtime_config.tools} + definitions_by_id: dict[str, TestCaseDefinition] = {} + has_image_input = False for row in ordered_cases: if row.input_mode != "fixed_script_text": - raise HTTPException(422, f"用例“{row.name}”不是可运行的固定文字模式") + raise HTTPException(422, f"用例“{row.name}”不是可运行的固定多模态模式") try: definition = TestCaseDefinition.model_validate(row.definition or {}) + await canonicalize_test_assets(session, definition) except ValueError as exc: raise HTTPException(422, f"用例“{row.name}”校验失败: {exc}") from exc + definitions_by_id[row.id] = definition + has_image_input = has_image_input or bool(image_asset_ids(definition)) for turn in definition.turns: for behavior in turn.behaviors: if behavior.type != "tool_call": @@ -159,6 +166,14 @@ async def create_test_run( "不属于当前被测助手", ) + if has_image_input: + try: + vision_enabled = validate_visual_runtime(runtime_config) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + if not vision_enabled: + raise HTTPException(422, "测试用例包含图片，但当前助手未开启视觉能力") + suite_ids = list(dict.fromkeys(row.suite_id for row in ordered_cases)) suites = ( await session.execute(select(TestSuite).where(TestSuite.id.in_(suite_ids))) @@ -192,7 +207,7 @@ async def create_test_run( ) session.add(run) for position, row in enumerate(ordered_cases): - definition = TestCaseDefinition.model_validate(row.definition or {}) + definition = definitions_by_id[row.id] snapshot = { "id": row.id, "suiteId": row.suite_id, diff --git a/backend/services/image_processing.py b/backend/services/image_processing.py new file mode 100644 index 0000000..c2ba975 --- /dev/null +++ b/backend/services/image_processing.py @@ -0,0 +1,41 @@ +"""Shared validation and normalization for user-supplied images.""" + +from __future__ import annotations + +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + +import settings + + +def normalize_image(data: bytes) -> tuple[bytes, int, int]: + """Return a bounded, orientation-corrected JPEG safe for model input.""" + + try: + source = Image.open(BytesIO(data)) + width, height = source.size + if width <= 0 or height <= 0: + raise ValueError("图片尺寸无效") + if width * height > settings.INPUT_IMAGE_MAX_PIXELS: + raise ValueError("图片像素尺寸过大") + source.load() + except (UnidentifiedImageError, OSError) as exc: + raise ValueError("文件不是可识别的图片") from exc + + image = ImageOps.exif_transpose(source) + image.thumbnail( + (settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE), + Image.Resampling.LANCZOS, + ) + if image.mode in {"RGBA", "LA"} or "transparency" in image.info: + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, "white") + background.paste(rgba, mask=rgba.getchannel("A")) + image = background + else: + image = image.convert("RGB") + + buffer = BytesIO() + image.save(buffer, format="JPEG", quality=85, optimize=True) + return buffer.getvalue(), image.width, image.height diff --git a/backend/services/input_assets.py b/backend/services/input_assets.py index bbba40b..6c4480a 100644 --- a/backend/services/input_assets.py +++ b/backend/services/input_assets.py @@ -8,12 +8,10 @@ import hmac import json import time from dataclasses import dataclass -from io import BytesIO from uuid import uuid4 -from PIL import Image, ImageOps, UnidentifiedImageError - import settings +from services.image_processing import normalize_image from services.object_storage import delete_object, get_object, put_object @@ -46,38 +44,8 @@ def _signature(payload: str) -> str: return _b64encode(digest) -def _normalize_image(data: bytes) -> tuple[bytes, int, int]: - try: - source = Image.open(BytesIO(data)) - width, height = source.size - if width <= 0 or height <= 0: - raise ValueError("图片尺寸无效") - if width * height > settings.INPUT_IMAGE_MAX_PIXELS: - raise ValueError("图片像素尺寸过大") - source.load() - except (UnidentifiedImageError, OSError) as exc: - raise ValueError("文件不是可识别的图片") from exc - - image = ImageOps.exif_transpose(source) - image.thumbnail( - (settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE), - Image.Resampling.LANCZOS, - ) - if image.mode in {"RGBA", "LA"} or "transparency" in image.info: - rgba = image.convert("RGBA") - background = Image.new("RGB", rgba.size, "white") - background.paste(rgba, mask=rgba.getchannel("A")) - image = background - else: - image = image.convert("RGB") - - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=85, optimize=True) - return buffer.getvalue(), image.width, image.height - - def store_input_image(data: bytes) -> StoredInputImage: - normalized, width, height = _normalize_image(data) + normalized, width, height = normalize_image(data) key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg" put_object(key, normalized, "image/jpeg") expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index 2227bb4..645b7e1 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -34,12 +34,11 @@ from services.tool_policy import policy_for_tool from services.vision import ( VISION_TOOL_NAME, analyze_image_with_vision_model, - config_with_main_llm_as_vision, - config_with_vision_resource, image_data_uri, image_frame_from_jpeg, image_jpeg_bytes, resolve_vision_question, + resolve_vision_input, vision_system_hint, vision_tool_schema_parts, ) @@ -151,30 +150,6 @@ def _require(value: str, label: str) -> str: raise ValueError(f"缺少模型资源配置: {label}") -def _vision_uses_main_llm(cfg: AssistantConfig) -> bool: - """模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。""" - return not cfg.vision_model_resource_id and cfg.llm_support_image_input - - -def _workflow_vision_uses_main_llm( - cfg: AssistantConfig, - scope: dict[str, Any], -) -> bool: - """Resolve native versus auxiliary vision for the active Workflow Agent.""" - if not scope.get("enabled"): - raise ValueError("当前 Workflow Agent 节点未启用视觉能力") - if scope.get("vision_model_resource_id"): - return False - - llm_resource_id = str(scope.get("llm_resource_id") or "") - resource = cfg.workflow_model_resources.get(llm_resource_id) - if not resource: - raise ValueError(f"当前 Workflow Agent 的 LLM 资源未加载:{llm_resource_id}") - if not resource.support_image_input: - raise ValueError("当前 Workflow Agent 的 LLM 不支持图片输入") - return True - - def _multimodal_user_input_frame( image_frame: UserImageRawFrame, prompt_text: str, @@ -485,7 +460,7 @@ async def run_pipeline( vision_native_mode = ( vision_enabled and cfg.type != "workflow" - and _vision_uses_main_llm(cfg) + and resolve_vision_input(cfg).native ) vision_state: dict[str, str | None] = {"client_id": None} workflow_vision_scope: dict[str, Any] = { @@ -497,9 +472,9 @@ async def run_pipeline( } def active_vision_uses_main_llm() -> bool: - if cfg.type == "workflow": - return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope) - return vision_native_mode + if cfg.type != "workflow": + return vision_native_mode + return resolve_vision_input(cfg, workflow_vision_scope).native vision_description, vision_properties, vision_required = vision_tool_schema_parts( cfg.vision_prompt_mode, cfg.vision_prompt, @@ -609,29 +584,8 @@ async def run_pipeline( workflow_vision_function = None def active_vision_config() -> AssistantConfig: - if cfg.type != "workflow": - return cfg - if not workflow_vision_scope.get("enabled"): - raise ValueError("当前 Workflow Agent 节点未启用视觉能力") - vision_resource_id = str( - workflow_vision_scope.get("vision_model_resource_id") or "" - ) - llm_resource_id = str(workflow_vision_scope.get("llm_resource_id") or "") - resource_id = vision_resource_id or llm_resource_id - resource = cfg.workflow_model_resources.get(resource_id) - if resource: - result = config_with_vision_resource(cfg, resource) - elif not vision_resource_id: - result = config_with_main_llm_as_vision(cfg) - else: - raise ValueError(f"视觉模型资源未加载:{vision_resource_id}") - result.vision_prompt_mode = str( - workflow_vision_scope.get("vision_prompt_mode") or "runtime" - ) - result.vision_prompt = str( - workflow_vision_scope.get("vision_prompt") or "" - ).strip() - return result + scope = workflow_vision_scope if cfg.type == "workflow" else None + return resolve_vision_input(cfg, scope).config if cfg.type == "workflow" and vision_enabled: async def native_flow_fetch_user_image(params: FunctionCallParams) -> None: diff --git a/backend/services/test_assets.py b/backend/services/test_assets.py new file mode 100644 index 0000000..f859e08 --- /dev/null +++ b/backend/services/test_assets.py @@ -0,0 +1,44 @@ +"""Persistent image fixtures used by repeatable assistant test cases.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from services.image_processing import normalize_image +from services.object_storage import delete_object, get_object, put_object + + +TEST_ASSET_PREFIX = "test-assets/" + + +@dataclass(frozen=True) +class StoredTestImage: + storage_key: str + width: int + height: int + size_bytes: int + mime_type: str = "image/jpeg" + + +def store_test_image(asset_id: str, data: bytes) -> StoredTestImage: + normalized, width, height = normalize_image(data) + key = f"{TEST_ASSET_PREFIX}{asset_id}.jpg" + put_object(key, normalized, "image/jpeg") + return StoredTestImage( + storage_key=key, + width=width, + height=height, + size_bytes=len(normalized), + ) + + +def read_test_image(storage_key: str) -> bytes: + if not storage_key.startswith(TEST_ASSET_PREFIX): + raise ValueError("测试图片存储位置无效") + return get_object(storage_key) + + +def delete_test_image(storage_key: str) -> None: + if not storage_key.startswith(TEST_ASSET_PREFIX): + raise ValueError("测试图片存储位置无效") + delete_object(storage_key) diff --git a/backend/services/test_case_assets.py b/backend/services/test_case_assets.py new file mode 100644 index 0000000..30544d8 --- /dev/null +++ b/backend/services/test_case_assets.py @@ -0,0 +1,72 @@ +"""Validation and loading helpers for image fixtures in test definitions.""" + +from __future__ import annotations + +import asyncio + +from db.models import TestAsset +from services.test_assets import read_test_image +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from test_schemas import TestCaseDefinition + + +def image_asset_ids(definition: TestCaseDefinition) -> set[str]: + return { + turn.image.asset_id + for turn in definition.turns + if turn.image is not None + } + + +async def canonicalize_test_assets( + session: AsyncSession, + definition: TestCaseDefinition, +) -> None: + """Validate references and replace client metadata with authoritative values.""" + + asset_ids = image_asset_ids(definition) + if not asset_ids: + return + rows = ( + await session.execute(select(TestAsset).where(TestAsset.id.in_(asset_ids))) + ).scalars().all() + by_id = {row.id: row for row in rows} + missing = sorted(asset_ids - by_id.keys()) + if missing: + raise ValueError("测试图片不存在: " + "、".join(missing)) + + for turn in definition.turns: + if turn.image is None: + continue + row = by_id[turn.image.asset_id] + turn.image.filename = row.original_name + turn.image.width = row.width + turn.image.height = row.height + turn.image.size_bytes = row.size_bytes + + +async def load_test_asset_bytes( + session: AsyncSession, + definition: TestCaseDefinition, +) -> dict[str, bytes]: + """Load immutable image fixtures once before a transportless test attempt.""" + + asset_ids = image_asset_ids(definition) + if not asset_ids: + return {} + rows = ( + await session.execute(select(TestAsset).where(TestAsset.id.in_(asset_ids))) + ).scalars().all() + by_id = {row.id: row for row in rows} + missing = sorted(asset_ids - by_id.keys()) + if missing: + raise ValueError("测试图片不存在: " + "、".join(missing)) + + pairs = await asyncio.gather( + *( + asyncio.to_thread(read_test_image, row.storage_key) + for row in rows + ) + ) + return {row.id: data for row, data in zip(rows, pairs, strict=True)} diff --git a/backend/services/test_runs/evaluator.py b/backend/services/test_runs/evaluator.py index 55e0d15..156d3ae 100644 --- a/backend/services/test_runs/evaluator.py +++ b/backend/services/test_runs/evaluator.py @@ -161,6 +161,7 @@ class EvaluationEngine: id=expected_turn.id, index=index, user_input=expected_turn.user_input, + image=expected_turn.image, assistant_reply="", tool_calls=[], runtime_records=[], @@ -174,6 +175,7 @@ class EvaluationEngine: id=raw.id, index=index, user_input=raw.user_input, + image=raw.image, assistant_reply=raw.assistant_reply, tool_calls=[item.as_result() for item in raw.tool_calls], runtime_records=list(raw.runtime_records), diff --git a/backend/services/test_runs/orchestrator.py b/backend/services/test_runs/orchestrator.py index ee78b1d..63d52bc 100644 --- a/backend/services/test_runs/orchestrator.py +++ b/backend/services/test_runs/orchestrator.py @@ -14,6 +14,7 @@ from db.session import SessionLocal from models import RuntimeModelResource from services.config_resolver import resolve_runtime_config from services.runtime_variables import prepare_dynamic_config +from services.test_case_assets import load_test_asset_bytes from services.test_runs.errors import TestExecutionError from services.test_runs.evaluator import EvaluationEngine from services.test_runs.text_runner import ( @@ -178,6 +179,19 @@ class TestRunOrchestrator: snapshot = dict(row.case_snapshot or {}) definition = TestCaseDefinition.model_validate(snapshot) max_attempts = row.max_attempts + try: + image_assets = await load_test_asset_bytes(session, definition) + except Exception as exc: + await self._mark_error( + run_case_id, + TestExecutionError( + code="TEST_IMAGE_READ_FAILED", + message=str(exc) or "测试图片无法读取", + stage="pipeline", + retryable=True, + ), + ) + return for attempt in range(1, max_attempts + 1): if control.stop_event.is_set(): @@ -212,6 +226,7 @@ class TestRunOrchestrator: attempt_cfg, definition, timeout_seconds=timeout_seconds, + image_assets=image_assets, on_turn_complete=save_turn_progress, on_startup_complete=save_startup_progress, ) @@ -346,6 +361,11 @@ class TestRunOrchestrator: "id": turn.id, "index": turn.index, "userInput": turn.user_input, + "image": ( + turn.image.model_dump(mode="json", by_alias=True) + if turn.image is not None + else None + ), "assistantReply": turn.assistant_reply, "toolCalls": [item.as_result() for item in turn.tool_calls], "runtimeRecords": list(turn.runtime_records), diff --git a/backend/services/test_runs/text_runner.py b/backend/services/test_runs/text_runner.py index 5555692..45c232d 100644 --- a/backend/services/test_runs/text_runner.py +++ b/backend/services/test_runs/text_runner.py @@ -55,7 +55,8 @@ from services.pipecat.service_factory import config_with_resource from services.pipecat.workflow_services import build_workflow_llm_switcher from services.test_runs.errors import TestExecutionError from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort -from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition +from services.vision import build_image_user_message +from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef IDLE_SETTLE_SECONDS = 0.2 @@ -104,6 +105,7 @@ class RawTurnResult: user_input: str assistant_reply: str tool_calls: list[RawToolCall] + image: TestImageRef | None = None runtime_records: list[dict[str, Any]] = field(default_factory=list) @@ -359,6 +361,31 @@ def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]: return messages +def _transcript_content(message: dict[str, Any]) -> str: + """Keep evaluator transcripts readable without persisting image data URIs.""" + + content = message.get("content") + if isinstance(content, str): + return content.strip() + if not isinstance(content, list): + return "" + text_parts = [ + str(part.get("text") or "").strip() + for part in content + if isinstance(part, dict) + and part.get("type") in {"text", "input_text"} + and str(part.get("text") or "").strip() + ] + has_image = any( + isinstance(part, dict) + and part.get("type") in {"image_url", "input_image"} + for part in content + ) + if has_image: + text_parts.append("[用户提交了一张测试图片]") + return "\n".join(text_parts).strip() + + async def _wait_for_quiescence( capture: _TextCaptureProcessor, runner_task: asyncio.Task[None], @@ -423,6 +450,7 @@ class TextPipelineRunner: definition: TestCaseDefinition, *, timeout_seconds: int, + image_assets: dict[str, bytes] | None = None, on_turn_complete: TurnCompleteCallback | None = None, on_startup_complete: StartupCompleteCallback | None = None, ) -> TextCaseResult: @@ -431,6 +459,7 @@ class TextPipelineRunner: return await self._run( cfg, definition, + image_assets=image_assets or {}, on_turn_complete=on_turn_complete, on_startup_complete=on_startup_complete, ) @@ -447,6 +476,7 @@ class TextPipelineRunner: cfg: AssistantConfig, definition: TestCaseDefinition, *, + image_assets: dict[str, bytes], on_turn_complete: TurnCompleteCallback | None, on_startup_complete: StartupCompleteCallback | None, ) -> TextCaseResult: @@ -545,6 +575,7 @@ class TextPipelineRunner: executor_holder: dict[str, MockToolExecutor] = {} text_client_tools = TextClientToolPort() + workflow_vision_scope: dict[str, Any] = {"enabled": False} def create_test_executor(store) -> MockToolExecutor: executor = MockToolExecutor(store) @@ -699,7 +730,7 @@ class TextPipelineRunner: transport=None, switch_services=switch_services, set_knowledge_scope=knowledge.set_scope, - set_vision_scope=lambda _scope: None, + set_vision_scope=lambda scope: workflow_vision_scope.update(scope), vision_function=None, set_input_enabled=lambda _enabled: None, apply_turn_config=lambda _enabled, _config: asyncio.sleep(0), @@ -809,26 +840,32 @@ class TextPipelineRunner: retryable=False, ) result = await self._run_turn( + cfg, worker, runner_task, capture, executor, turn, index, + image_assets=image_assets, + workflow_vision_scope=workflow_vision_scope, ) results.append(result) if on_turn_complete is not None: await on_turn_complete(result) - transcript = [ - { - "role": str(message.get("role") or ""), - "content": str(message.get("content") or ""), - } - for message in context.get_messages() - if message.get("role") in {"user", "assistant"} - and str(message.get("content") or "").strip() - ] + transcript = [] + for message in context.get_messages(): + if message.get("role") not in {"user", "assistant"}: + continue + content = _transcript_content(message) + if content: + transcript.append( + { + "role": str(message.get("role") or ""), + "content": content, + } + ) return TextCaseResult( turns=results, transcript=transcript, @@ -853,12 +890,16 @@ class TextPipelineRunner: async def _run_turn( self, + cfg: AssistantConfig, worker: PipelineWorker, runner_task: asyncio.Task[None], capture: _TextCaptureProcessor, executor: MockToolExecutor, turn: FixedInputTurn, index: int, + *, + image_assets: dict[str, bytes], + workflow_vision_scope: dict[str, Any], ) -> RawTurnResult: tool_behaviors = [ behavior for behavior in turn.behaviors if behavior.type == "tool_call" @@ -868,11 +909,37 @@ class TextPipelineRunner: output_marker = len(capture.outputs) tool_marker = len(capture.tool_calls) runtime_record_marker = len(capture.runtime_records) + user_message: dict[str, Any] = { + "role": "user", + "content": turn.user_input, + } + if turn.image is not None: + image_bytes = image_assets.get(turn.image.asset_id) + if image_bytes is None: + raise TestExecutionError( + code="TEST_IMAGE_MISSING", + message=f"测试图片不存在: {turn.image.asset_id}", + stage="pipeline", + retryable=False, + ) + try: + user_message = await build_image_user_message( + cfg, + image_bytes, + turn.user_input, + workflow_scope=( + workflow_vision_scope if cfg.type == "workflow" else None + ), + ) + except ValueError as exc: + raise TestExecutionError( + code="TEST_IMAGE_UNSUPPORTED", + message=str(exc), + stage="model", + retryable=False, + ) from exc await worker.queue_frame( - LLMMessagesAppendFrame( - messages=[{"role": "user", "content": turn.user_input}], - run_llm=True, - ) + LLMMessagesAppendFrame(messages=[user_message], run_llm=True) ) await _wait_for_quiescence(capture, runner_task, activity_marker) if executor.unmocked_calls: @@ -886,6 +953,7 @@ class TextPipelineRunner: id=turn.id, index=index, user_input=turn.user_input, + image=turn.image, assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(), tool_calls=list(capture.tool_calls[tool_marker:]), runtime_records=list( diff --git a/backend/services/vision.py b/backend/services/vision.py index d038c6b..08ac1aa 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -4,6 +4,7 @@ from __future__ import annotations import asyncio import base64 +from dataclasses import dataclass from io import BytesIO from typing import Any @@ -30,6 +31,14 @@ VISION_ANALYSIS_SYSTEM_PROMPT = ( ) +@dataclass(frozen=True) +class VisionInputResolution: + """The active model path for one explicit user image turn.""" + + native: bool + config: AssistantConfig + + def _require(value: str, label: str) -> str: if value: return value @@ -246,3 +255,83 @@ def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig: result.vision_llm_api_key = cfg.llm_api_key result.vision_llm_base_url = cfg.llm_base_url return result + + +def resolve_vision_input( + cfg: AssistantConfig, + workflow_scope: dict[str, Any] | None = None, +) -> VisionInputResolution: + """Resolve native versus auxiliary vision using the active runtime scope.""" + + if cfg.type != "workflow": + if not cfg.vision_enabled: + raise ValueError("当前助手未开启视觉能力") + if cfg.vision_model_resource_id: + if not cfg.vision_llm_support_image_input: + raise ValueError("当前视觉模型不支持图片输入") + return VisionInputResolution(native=False, config=cfg) + if not cfg.llm_support_image_input: + raise ValueError("当前大语言模型不支持图片输入") + return VisionInputResolution(native=True, config=cfg) + + scope = workflow_scope or {} + if not scope.get("enabled"): + raise ValueError("当前 Workflow Agent 节点未启用视觉能力") + vision_resource_id = str(scope.get("vision_model_resource_id") or "") + llm_resource_id = str(scope.get("llm_resource_id") or "") + resource_id = vision_resource_id or llm_resource_id + resource = cfg.workflow_model_resources.get(resource_id) + if resource is not None: + if not resource.support_image_input: + raise ValueError(f"当前视觉模型不支持图片输入: {resource.id}") + result = config_with_vision_resource(cfg, resource) + elif not vision_resource_id: + result = config_with_main_llm_as_vision(cfg) + else: + raise ValueError(f"视觉模型资源未加载: {vision_resource_id}") + + result.vision_prompt_mode = normalize_vision_prompt_mode( + scope.get("vision_prompt_mode") + ) + result.vision_prompt = str(scope.get("vision_prompt") or "").strip() + return VisionInputResolution( + native=not bool(vision_resource_id), + config=result, + ) + + +async def build_image_user_message( + cfg: AssistantConfig, + image_bytes: bytes, + prompt_text: str, + *, + workflow_scope: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Build the same model-facing user message used by an explicit photo turn.""" + + question = prompt_text.strip() or "请根据用户刚提交的图片进行回复。" + frame = await asyncio.to_thread(image_frame_from_jpeg, image_bytes) + resolution = resolve_vision_input(cfg, workflow_scope) + if resolution.native: + data_uri = await asyncio.to_thread(image_data_uri, frame) + return { + "role": "user", + "content": [ + {"type": "text", "text": question}, + {"type": "image_url", "image_url": {"url": data_uri}}, + ], + } + + observation = await analyze_image_with_vision_model( + resolution.config, + frame, + question, + ) + return { + "role": "user", + "content": ( + f"{question}\n\n" + "[视觉模型对用户刚提交图片的观察]\n" + f"{observation or '视觉模型没有返回有效观察结果。'}" + ), + } diff --git a/backend/test_schemas.py b/backend/test_schemas.py index 60a7880..1fb1c77 100644 --- a/backend/test_schemas.py +++ b/backend/test_schemas.py @@ -143,20 +143,36 @@ ExpectedBehavior = Annotated[ ] -class FixedInputTurn(CamelModel): - id: str = Field(min_length=1, max_length=128) - user_input: str = Field(min_length=1, max_length=20_000) - behaviors: list[ExpectedBehavior] = Field(default_factory=list, max_length=100) +class TestImageRef(CamelModel): + asset_id: str = Field(min_length=1, max_length=40) + filename: str = Field(default="image.jpg", max_length=255) + width: int = Field(ge=1) + height: int = Field(ge=1) + size_bytes: int = Field(default=0, ge=0) - @field_validator("user_input") + @field_validator("asset_id", "filename") @classmethod - def strip_user_input(cls, value: str) -> str: + def strip_image_text(cls, value: str) -> str: value = value.strip() if not value: - raise ValueError("用户输入不能为空") + raise ValueError("图片资源信息不完整") return value +class FixedInputTurn(CamelModel): + id: str = Field(min_length=1, max_length=128) + user_input: str = Field(default="", max_length=20_000) + image: TestImageRef | None = None + behaviors: list[ExpectedBehavior] = Field(default_factory=list, max_length=100) + + @model_validator(mode="after") + def validate_input(self): + self.user_input = self.user_input.strip() + if not self.user_input and self.image is None: + raise ValueError("用户输入和图片至少填写一项") + return self + + class OverallCriterion(CamelModel): id: str = Field(min_length=1, max_length=128) type: Literal["llm"] = "llm" @@ -272,7 +288,7 @@ class TestCaseWrite(CamelModel): @model_validator(mode="after") def validate_mvp_definition(self): if self.input_mode != "fixed_script_text": - raise ValueError("第一版只支持固定脚本 · 文字") + raise ValueError("第一版只支持固定脚本 · 多模态") TestCaseDefinition( context_turns=self.context_turns, turns=self.turns, @@ -362,6 +378,7 @@ class BatchTurnResult(CamelModel): id: str index: int user_input: str + image: TestImageRef | None = None assistant_reply: str tool_calls: list[BatchToolCallRecord] = Field(default_factory=list) runtime_records: list[dict[str, Any]] = Field(default_factory=list) diff --git a/backend/tests/test_pipeline_knowledge.py b/backend/tests/test_pipeline_knowledge.py index de32af2..f5cbdfb 100644 --- a/backend/tests/test_pipeline_knowledge.py +++ b/backend/tests/test_pipeline_knowledge.py @@ -9,8 +9,8 @@ from services.pipecat.pipeline import ( KnowledgeRetrievalProcessor, UserTurnRoutingProcessor, _knowledge_tool_description, - _workflow_vision_uses_main_llm, ) +from services.vision import resolve_vision_input class KnowledgeToolDescriptionTest(unittest.TestCase): @@ -159,29 +159,36 @@ class WorkflowVisionModeTest(unittest.TestCase): capability="LLM", interface_type="openai-llm", support_image_input=True, - ) + ), + "auxiliary_vision": RuntimeModelResource( + id="auxiliary_vision", + name="独立视觉模型", + capability="LLM", + interface_type="openai-llm", + support_image_input=True, + ), }, ) self.assertTrue( - _workflow_vision_uses_main_llm( + resolve_vision_input( cfg, { "enabled": True, "llm_resource_id": "agent_llm", "vision_model_resource_id": None, }, - ) + ).native ) self.assertFalse( - _workflow_vision_uses_main_llm( + resolve_vision_input( cfg, { "enabled": True, "llm_resource_id": "agent_llm", "vision_model_resource_id": "auxiliary_vision", }, - ) + ).native ) def test_rejects_a_non_visual_active_agent_llm(self): @@ -199,7 +206,7 @@ class WorkflowVisionModeTest(unittest.TestCase): ) with self.assertRaisesRegex(ValueError, "不支持图片输入"): - _workflow_vision_uses_main_llm( + resolve_vision_input( cfg, { "enabled": True, diff --git a/backend/tests/test_test_image_inputs.py b/backend/tests/test_test_image_inputs.py new file mode 100644 index 0000000..1fe7fbe --- /dev/null +++ b/backend/tests/test_test_image_inputs.py @@ -0,0 +1,118 @@ +from io import BytesIO +import unittest +from unittest.mock import AsyncMock, patch + +from PIL import Image +from pydantic import ValidationError + +from models import AssistantConfig +from services.test_assets import read_test_image, store_test_image +from services.test_runs.text_runner import _transcript_content +from services.vision import build_image_user_message +from test_schemas import FixedInputTurn, TestImageRef + + +def _jpeg_bytes() -> bytes: + buffer = BytesIO() + Image.new("RGB", (12, 8), (220, 40, 40)).save(buffer, format="JPEG") + return buffer.getvalue() + + +def _image_ref() -> TestImageRef: + return TestImageRef( + asset_id="ta_fixture", + filename="fixture.jpg", + width=12, + height=8, + size_bytes=128, + ) + + +class TestImageSchemaTests(unittest.TestCase): + def test_allows_image_only_turn(self): + turn = FixedInputTurn(id="turn_1", image=_image_ref()) + + self.assertEqual(turn.user_input, "") + self.assertEqual(turn.image.asset_id, "ta_fixture") + + def test_rejects_turn_without_text_or_image(self): + with self.assertRaisesRegex(ValidationError, "至少填写一项"): + FixedInputTurn(id="turn_1") + + +class PersistentTestAssetTests(unittest.TestCase): + def test_normalizes_and_stores_under_test_prefix(self): + with patch("services.test_assets.put_object") as put_object: + stored = store_test_image("ta_fixture", _jpeg_bytes()) + + self.assertEqual(stored.storage_key, "test-assets/ta_fixture.jpg") + self.assertEqual((stored.width, stored.height), (12, 8)) + self.assertEqual(put_object.call_args.args[0], stored.storage_key) + self.assertEqual(put_object.call_args.args[2], "image/jpeg") + + def test_reads_only_test_asset_keys(self): + with patch("services.test_assets.get_object", return_value=b"jpeg"): + self.assertEqual(read_test_image("test-assets/ta_fixture.jpg"), b"jpeg") + with self.assertRaisesRegex(ValueError, "存储位置无效"): + read_test_image("conversation-inputs/unsafe.jpg") + + +class ImageUserMessageTests(unittest.IsolatedAsyncioTestCase): + async def test_native_model_receives_multimodal_message(self): + message = await build_image_user_message( + AssistantConfig( + type="prompt", + vision_enabled=True, + llm_support_image_input=True, + ), + _jpeg_bytes(), + "这是什么颜色？", + ) + + self.assertEqual(message["role"], "user") + self.assertEqual(message["content"][0]["text"], "这是什么颜色？") + self.assertTrue( + message["content"][1]["image_url"]["url"].startswith( + "data:image/jpeg;base64," + ) + ) + + async def test_auxiliary_model_observation_becomes_text_context(self): + with patch( + "services.vision.analyze_image_with_vision_model", + new=AsyncMock(return_value="画面中是一个红色方块。"), + ): + message = await build_image_user_message( + AssistantConfig( + type="prompt", + vision_enabled=True, + vision_model_resource_id="vision_model", + vision_llm_support_image_input=True, + ), + _jpeg_bytes(), + "画面里有什么？", + ) + + self.assertIsInstance(message["content"], str) + self.assertIn("红色方块", message["content"]) + + def test_transcript_does_not_persist_image_data(self): + content = _transcript_content( + { + "role": "user", + "content": [ + {"type": "text", "text": "请看图"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,secret"}, + }, + ], + } + ) + + self.assertEqual(content, "请看图\n[用户提交了一张测试图片]") + self.assertNotIn("base64", content) + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/components/assistant-editor/debug-auto-test.tsx b/frontend/src/components/assistant-editor/debug-auto-test.tsx index 193933f..c769528 100644 --- a/frontend/src/components/assistant-editor/debug-auto-test.tsx +++ b/frontend/src/components/assistant-editor/debug-auto-test.tsx @@ -51,6 +51,7 @@ import { import { batchRunsApi, modelResourcesApi, + testAssetsApi, testCasesApi, testSuitesApi, type ModelResource, @@ -101,7 +102,7 @@ function isRunFinished(run: BatchRunSnapshot): boolean { function runnableMessage(item: TestCase): string { if (item.inputMode !== "fixed_script_text") { - return "当前自动测试仅支持固定脚本 · 文字"; + return "当前自动测试仅支持固定脚本 · 多模态"; } return getTestCaseValidationMessage(item) ?? ""; } @@ -653,6 +654,16 @@ function AutoTestTimeline({ role: "user", label: `固定输入 · ${turn.index + 1} / ${maxTurns}`, content: turn.userInput, + attachments: turn.image + ? [ + { + id: turn.image.assetId, + type: "image", + url: testAssetsApi.contentUrl(turn.image.assetId), + alt: turn.image.filename, + }, + ] + : undefined, }, }); items.push({ diff --git a/frontend/src/components/batch-test/batch-case-detail-drawer.tsx b/frontend/src/components/batch-test/batch-case-detail-drawer.tsx index ef8f62f..ddd3ed5 100644 --- a/frontend/src/components/batch-test/batch-case-detail-drawer.tsx +++ b/frontend/src/components/batch-test/batch-case-detail-drawer.tsx @@ -28,6 +28,7 @@ import { type BatchToolCallRecord, type BatchTurnResult, } from "@/data/batch-run"; +import { testAssetsApi } from "@/lib/api"; import { cn } from "@/lib/utils"; type BatchCaseDetailDrawerProps = { @@ -98,7 +99,7 @@ export function BatchCaseDetailDrawer({

执行契约

- 正在执行固定文字脚本；完成后将展示每一轮回复、工具调用和全部评估标准。 + 正在执行固定多模态脚本；完成后将展示每轮回复、工具调用和评估标准。

@@ -235,9 +236,19 @@ function TurnResultCard({ turn }: { turn: BatchTurnResult }) {

用户输入 -

- {turn.userInput} -

+
+ {turn.image ? ( + // eslint-disable-next-line @next/next/no-img-element + {turn.image.filename} + ) : null} + {turn.userInput ? ( +

{turn.userInput}

+ ) : null} +
Agent 回复

{turn.assistantReply} diff --git a/frontend/src/components/pages/TestCasesPage.tsx b/frontend/src/components/pages/TestCasesPage.tsx index 4f8ef69..5fe9b36 100644 --- a/frontend/src/components/pages/TestCasesPage.tsx +++ b/frontend/src/components/pages/TestCasesPage.tsx @@ -112,6 +112,8 @@ function SuiteListView() { } useEffect(() => { + // Initial API hydration is intentionally owned by this mount effect. + // eslint-disable-next-line react-hooks/set-state-in-effect void reloadSuites(); }, []); @@ -507,6 +509,8 @@ function SuiteDetailView({ } useEffect(() => { + // Suite changes remount this view; hydrate the selected case once. + // eslint-disable-next-line react-hooks/set-state-in-effect void reload(initialCaseId ?? null); // suiteId changes by remount; the initial request only runs once per view. // eslint-disable-next-line react-hooks/exhaustive-deps @@ -931,7 +935,7 @@ function SuiteDetailView({ {TEST_CASE_INPUT_MODE_LABEL[draft.inputMode]} · 开发中

- 当前仅支持「固定脚本 · 文字」编辑器，其它输入模式稍后接入。 + 当前仅支持「固定脚本 · 多模态」编辑器，其它输入模式稍后接入。

)} diff --git a/frontend/src/components/test-cases/input-mode-picker.tsx b/frontend/src/components/test-cases/input-mode-picker.tsx index 6f2de90..beff4d9 100644 --- a/frontend/src/components/test-cases/input-mode-picker.tsx +++ b/frontend/src/components/test-cases/input-mode-picker.tsx @@ -37,8 +37,8 @@ const FIXED_SCRIPT_OPTIONS: { }[] = [ { value: "fixed_script_text", - title: "文字", - description: "使用固定文字逐轮测试", + title: "文字与图片", + description: "使用固定文字或图片逐轮测试", icon: Type, available: true, }, diff --git a/frontend/src/components/test-cases/next-reply-editor.tsx b/frontend/src/components/test-cases/next-reply-editor.tsx index fe600a0..2585f60 100644 --- a/frontend/src/components/test-cases/next-reply-editor.tsx +++ b/frontend/src/components/test-cases/next-reply-editor.tsx @@ -2,7 +2,7 @@ /** * 测试用例编辑器主体： - * 对话上下文 → 固定文字脚本轮次 → 整体评估标准。 + * 对话上下文 → 固定多模态脚本轮次 → 整体评估标准。 */ import { @@ -26,6 +26,8 @@ import { ChevronDown, ChevronUp, GripVertical, + ImagePlus, + Loader2, MessageSquareText, MessagesSquare, Plus, @@ -71,12 +73,13 @@ import { type KeywordMatchMode, type OverallCriterion, type ReplyExpectedBehavior, + type TestImageRef, type TestCaseInputMode, type ToolCallExpectedBehavior, type ToolParamAssertion, type ToolParamMatchMode, } from "@/data/test-suites"; -import { toolsApi, type Tool } from "@/lib/api"; +import { testAssetsApi, toolsApi, type Tool } from "@/lib/api"; import { cn } from "@/lib/utils"; export type CaseEditorDraft = { @@ -637,6 +640,9 @@ export function NextReplyEditorBody({ onChangeUserInput={(userInput) => updateTurn(turn.id, { userInput }) } + onChangeImage={(image) => + updateTurn(turn.id, { image }) + } onAddReply={() => addBehavior(turn.id, createReplyBehavior("llm")) } @@ -812,6 +818,7 @@ function SortableFixedInputTurn({ expandedBehaviorIds, onToggleBehavior, onChangeUserInput, + onChangeImage, onAddReply, onAddToolCall, onUpdateBehavior, @@ -825,6 +832,7 @@ function SortableFixedInputTurn({ expandedBehaviorIds: Set; onToggleBehavior: (behaviorId: string) => void; onChangeUserInput: (value: string) => void; + onChangeImage: (value: TestImageRef | null) => void; onAddReply: () => void; onAddToolCall: () => void; onUpdateBehavior: (behavior: ExpectedBehavior) => void; @@ -840,6 +848,34 @@ function SortableFixedInputTurn({ isDragging, } = useSortable({ id: turn.id }); const [addOpen, setAddOpen] = useState(false); + const fileInputRef = useRef(null); + const [uploadingImage, setUploadingImage] = useState(false); + const [imageError, setImageError] = useState(""); + + async function uploadImage(file: File | undefined) { + if (!file || uploadingImage) return; + if (!file.type.startsWith("image/")) { + setImageError("请选择图片文件"); + return; + } + setUploadingImage(true); + setImageError(""); + try { + const asset = await testAssetsApi.uploadImage(file); + onChangeImage({ + assetId: asset.id, + filename: asset.filename, + width: asset.width, + height: asset.height, + sizeBytes: asset.sizeBytes, + }); + } catch (error) { + setImageError(error instanceof Error ? error.message : "图片上传失败"); + } finally { + setUploadingImage(false); + if (fileInputRef.current) fileInputRef.current.value = ""; + } + } const style = { transform: CSS.Transform.toString(transform), @@ -888,17 +924,108 @@ function SortableFixedInputTurn({
-
+
{ + if (event.dataTransfer.types.includes("Files")) event.preventDefault(); + }} + onDrop={(event) => { + const file = Array.from(event.dataTransfer.files).find((item) => + item.type.startsWith("image/"), + ); + if (!file) return; + event.preventDefault(); + void uploadImage(file); + }} + >
用户输入