feat: support image inputs in assistant tests

backend/services/test_runs/text_runner.py

@@ -55,7 +55,8 @@ from services.pipecat.service_factory import config_with_resource
 from services.pipecat.workflow_services import build_workflow_llm_switcher
 from services.test_runs.errors import TestExecutionError
 from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
-from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition
+from services.vision import build_image_user_message
+from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef
 
 
 IDLE_SETTLE_SECONDS = 0.2
@@ -104,6 +105,7 @@ class RawTurnResult:
     user_input: str
     assistant_reply: str
     tool_calls: list[RawToolCall]
+    image: TestImageRef | None = None
     runtime_records: list[dict[str, Any]] = field(default_factory=list)
 
 
@@ -359,6 +361,31 @@ def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
     return messages
 
 
+def _transcript_content(message: dict[str, Any]) -> str:
+    """Keep evaluator transcripts readable without persisting image data URIs."""
+
+    content = message.get("content")
+    if isinstance(content, str):
+        return content.strip()
+    if not isinstance(content, list):
+        return ""
+    text_parts = [
+        str(part.get("text") or "").strip()
+        for part in content
+        if isinstance(part, dict)
+        and part.get("type") in {"text", "input_text"}
+        and str(part.get("text") or "").strip()
+    ]
+    has_image = any(
+        isinstance(part, dict)
+        and part.get("type") in {"image_url", "input_image"}
+        for part in content
+    )
+    if has_image:
+        text_parts.append("[用户提交了一张测试图片]")
+    return "\n".join(text_parts).strip()
+
+
 async def _wait_for_quiescence(
     capture: _TextCaptureProcessor,
     runner_task: asyncio.Task[None],
@@ -423,6 +450,7 @@ class TextPipelineRunner:
         definition: TestCaseDefinition,
         *,
         timeout_seconds: int,
+        image_assets: dict[str, bytes] | None = None,
         on_turn_complete: TurnCompleteCallback | None = None,
         on_startup_complete: StartupCompleteCallback | None = None,
     ) -> TextCaseResult:
@@ -431,6 +459,7 @@ class TextPipelineRunner:
                 return await self._run(
                     cfg,
                     definition,
+                    image_assets=image_assets or {},
                     on_turn_complete=on_turn_complete,
                     on_startup_complete=on_startup_complete,
                 )
@@ -447,6 +476,7 @@ class TextPipelineRunner:
         cfg: AssistantConfig,
         definition: TestCaseDefinition,
         *,
+        image_assets: dict[str, bytes],
         on_turn_complete: TurnCompleteCallback | None,
         on_startup_complete: StartupCompleteCallback | None,
     ) -> TextCaseResult:
@@ -545,6 +575,7 @@ class TextPipelineRunner:
 
         executor_holder: dict[str, MockToolExecutor] = {}
         text_client_tools = TextClientToolPort()
+        workflow_vision_scope: dict[str, Any] = {"enabled": False}
 
         def create_test_executor(store) -> MockToolExecutor:
             executor = MockToolExecutor(store)
@@ -699,7 +730,7 @@ class TextPipelineRunner:
                 transport=None,
                 switch_services=switch_services,
                 set_knowledge_scope=knowledge.set_scope,
-                set_vision_scope=lambda _scope: None,
+                set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
                 vision_function=None,
                 set_input_enabled=lambda _enabled: None,
                 apply_turn_config=lambda _enabled, _config: asyncio.sleep(0),
@@ -809,26 +840,32 @@ class TextPipelineRunner:
                         retryable=False,
                     )
                 result = await self._run_turn(
+                    cfg,
                     worker,
                     runner_task,
                     capture,
                     executor,
                     turn,
                     index,
+                    image_assets=image_assets,
+                    workflow_vision_scope=workflow_vision_scope,
                 )
                 results.append(result)
                 if on_turn_complete is not None:
                     await on_turn_complete(result)
 
-            transcript = [
-                {
-                    "role": str(message.get("role") or ""),
-                    "content": str(message.get("content") or ""),
-                }
-                for message in context.get_messages()
-                if message.get("role") in {"user", "assistant"}
-                and str(message.get("content") or "").strip()
-            ]
+            transcript = []
+            for message in context.get_messages():
+                if message.get("role") not in {"user", "assistant"}:
+                    continue
+                content = _transcript_content(message)
+                if content:
+                    transcript.append(
+                        {
+                            "role": str(message.get("role") or ""),
+                            "content": content,
+                        }
+                    )
             return TextCaseResult(
                 turns=results,
                 transcript=transcript,
@@ -853,12 +890,16 @@ class TextPipelineRunner:
 
     async def _run_turn(
         self,
+        cfg: AssistantConfig,
         worker: PipelineWorker,
         runner_task: asyncio.Task[None],
         capture: _TextCaptureProcessor,
         executor: MockToolExecutor,
         turn: FixedInputTurn,
         index: int,
+        *,
+        image_assets: dict[str, bytes],
+        workflow_vision_scope: dict[str, Any],
     ) -> RawTurnResult:
         tool_behaviors = [
             behavior for behavior in turn.behaviors if behavior.type == "tool_call"
@@ -868,11 +909,37 @@ class TextPipelineRunner:
         output_marker = len(capture.outputs)
         tool_marker = len(capture.tool_calls)
         runtime_record_marker = len(capture.runtime_records)
+        user_message: dict[str, Any] = {
+            "role": "user",
+            "content": turn.user_input,
+        }
+        if turn.image is not None:
+            image_bytes = image_assets.get(turn.image.asset_id)
+            if image_bytes is None:
+                raise TestExecutionError(
+                    code="TEST_IMAGE_MISSING",
+                    message=f"测试图片不存在: {turn.image.asset_id}",
+                    stage="pipeline",
+                    retryable=False,
+                )
+            try:
+                user_message = await build_image_user_message(
+                    cfg,
+                    image_bytes,
+                    turn.user_input,
+                    workflow_scope=(
+                        workflow_vision_scope if cfg.type == "workflow" else None
+                    ),
+                )
+            except ValueError as exc:
+                raise TestExecutionError(
+                    code="TEST_IMAGE_UNSUPPORTED",
+                    message=str(exc),
+                    stage="model",
+                    retryable=False,
+                ) from exc
         await worker.queue_frame(
-            LLMMessagesAppendFrame(
-                messages=[{"role": "user", "content": turn.user_input}],
-                run_llm=True,
-            )
+            LLMMessagesAppendFrame(messages=[user_message], run_llm=True)
         )
         await _wait_for_quiescence(capture, runner_task, activity_marker)
         if executor.unmocked_calls:
@@ -886,6 +953,7 @@ class TextPipelineRunner:
             id=turn.id,
             index=index,
             user_input=turn.user_input,
+            image=turn.image,
             assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
             tool_calls=list(capture.tool_calls[tool_marker:]),
             runtime_records=list(