"""Transportless Pipecat runner for one persisted fixed-text test case.""" from __future__ import annotations import asyncio import json import time from dataclasses import dataclass, field from typing import Any, Awaitable, Callable from uuid import uuid4 from loguru import logger from models import AssistantConfig from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, DataFrame, EndFrame, ErrorFrame, FunctionCallInProgressFrame, FunctionCallResultFrame, LLMContextFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, LLMMessagesAppendFrame, ManuallySwitchServiceFrame, OutputTransportMessageUrgentFrame, TTSSpeakFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.aggregators.llm_response_universal import ( LLMContextAggregatorPair, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.llm_service import FunctionCallParams from pipecat.workers.runner import WorkerRunner from db.session import SessionLocal from services.brains import BrainRuntime, build_brain from services.knowledge import search as search_knowledge from services.pipecat.call_lifecycle import ( CallEndCoordinator, FixedSpeechPlaybackMarkerFrame, call_end_trace_event, ) from services.pipecat.processors import ( KnowledgeRetrievalProcessor, UserTurnRoutingProcessor, ) from services.pipecat.service_factory import config_with_resource from services.pipecat.workflow_services import build_workflow_llm_switcher from services.test_runs.errors import TestExecutionError from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort from services.vision import build_image_user_message from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef IDLE_SETTLE_SECONDS = 0.2 WORKER_START_TIMEOUT_SECONDS = 5.0 INTERNAL_CANCEL_REASON = "text_test_case_complete" AUTOMATIC_KNOWLEDGE_HINT = ( "你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；" "回答资料事实时只根据检索内容，资料不足要明确说明。" ) ON_DEMAND_KNOWLEDGE_HINT = ( "你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，" "先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，" "资料不足要明确说明。" ) @dataclass class RawToolCall: id: str function_name: str arguments: Any result: Any = None outcome: str = "error" duration_ms: int = 0 _started_at: float = field(default_factory=time.monotonic, repr=False) def as_result(self) -> dict[str, Any]: return { "id": self.id, "functionName": self.function_name, "argumentsJson": json.dumps( self.arguments, ensure_ascii=False, indent=2, default=str ), "resultJson": json.dumps( self.result, ensure_ascii=False, indent=2, default=str ), "outcome": self.outcome, "durationMs": self.duration_ms, } @dataclass class RawTurnResult: id: str index: int user_input: str assistant_reply: str tool_calls: list[RawToolCall] image: TestImageRef | None = None runtime_records: list[dict[str, Any]] = field(default_factory=list) @dataclass class TextStartupResult: """Headless startup choices applied before the scripted user turns.""" greeting: str = "" assistant_outputs: list[str] = field(default_factory=list) auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list) runtime_records: list[dict[str, Any]] = field(default_factory=list) @dataclass class TextCaseResult: turns: list[RawTurnResult] transcript: list[dict[str, str]] startup: TextStartupResult = field(default_factory=TextStartupResult) TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]] StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]] @dataclass class _StartupBoundaryFrame(DataFrame): """Ordered marker queued behind every frame emitted by startup.""" reached: asyncio.Event @dataclass class _ResponseWindow: active_llm: int = 0 active_assistant: int = 0 pending_tools: set[str] = field(default_factory=set) activity_count: int = 0 last_activity_at: float = field(default_factory=time.monotonic) def touch(self) -> None: self.activity_count += 1 self.last_activity_at = time.monotonic() @property def idle(self) -> bool: return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools def _opening_speech_from_message(message: Any) -> str: """Extract only the transcript emitted by Prompt's opening Message stage.""" if ( not isinstance(message, dict) or message.get("type") != "transcript" or message.get("source") != "prompt-opening-speech" ): return "" return str(message.get("content") or "").strip() def _runtime_record_from_message(message: Any) -> dict[str, Any] | None: """Keep the same runtime-record envelope used by live conversation history.""" if ( not isinstance(message, dict) or message.get("type") != "workflow-event" or message.get("internalStage") is True ): return None return dict(message) class _TextCaptureProcessor(FrameProcessor): """Capture assistant/tool output and expose a reliable response boundary.""" def __init__(self, context: LLMContext): super().__init__() self.context = context self.window = _ResponseWindow() self.outputs: list[str] = [] self.opening_speech_outputs: list[str] = [] self.runtime_records: list[dict[str, Any]] = [] self.tool_calls: list[RawToolCall] = [] self._tool_by_id: dict[str, RawToolCall] = {} self.error: TestExecutionError | None = None self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None def fail(self, frame: ErrorFrame) -> None: """Release every wait even when Pipecat reports the error upstream.""" message = frame.error or "Pipeline 执行失败" unmocked_tool = "UNMOCKED_TOOL_CALL" in message self.error = TestExecutionError( code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR", message=message, stage=( "tool" if unmocked_tool else "model" if frame.processor else "pipeline" ), retryable=not unmocked_tool, ) self.window.pending_tools.clear() self.window.touch() async def process_frame(self, frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, TTSSpeakFrame): text = frame.text.strip() if text: self.outputs.append(text) if frame.append_to_context: self.context.add_message({"role": "assistant", "content": text}) self.window.touch() if self.on_fixed_speech_complete is not None: await self.on_fixed_speech_complete() return if isinstance(frame, OutputTransportMessageUrgentFrame): opening_speech = _opening_speech_from_message(frame.message) if opening_speech: self.opening_speech_outputs.append(opening_speech) runtime_record = _runtime_record_from_message(frame.message) if runtime_record is not None: self.runtime_records.append(runtime_record) if isinstance(frame, FixedSpeechPlaybackMarkerFrame): await frame.completion.mark_played() self.window.touch() return if isinstance(frame, LLMFullResponseStartFrame): self.window.active_llm += 1 self.window.touch() elif isinstance(frame, LLMFullResponseEndFrame): self.window.active_llm = max(0, self.window.active_llm - 1) self.window.touch() await self.push_frame(frame, direction) # There is no TTS/output transport. This releases provider/tool # follow-up logic that normally waits for a speaking boundary. await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM) return elif isinstance(frame, FunctionCallInProgressFrame): call = RawToolCall( id=frame.tool_call_id, function_name=frame.function_name, arguments=frame.arguments or {}, ) self.tool_calls.append(call) self._tool_by_id[call.id] = call self.window.pending_tools.add(call.id) self.window.touch() elif isinstance(frame, FunctionCallResultFrame): call = self._tool_by_id.get(frame.tool_call_id) if call is None: call = RawToolCall( id=frame.tool_call_id, function_name=frame.function_name, arguments=frame.arguments or {}, ) self.tool_calls.append(call) self._tool_by_id[call.id] = call call.result = frame.result call.duration_ms = max(0, round((time.monotonic() - call._started_at) * 1000)) status = frame.result.get("status") if isinstance(frame.result, dict) else None call.outcome = "error" if status in {"error", "failed", "timeout"} else "success" self.window.pending_tools.discard(call.id) self.window.touch() elif isinstance(frame, ErrorFrame): self.fail(frame) elif isinstance(frame, _StartupBoundaryFrame): frame.reached.set() self.window.touch() elif isinstance(frame, (EndFrame, CancelFrame)): self.window.touch() await self.push_frame(frame, direction) def _resolve_startup_greeting( shared_greeting: str, opening_speech_outputs: list[str], ) -> str: """Return the opening speech shown by the headless run. Prompt confirmation mode deliberately suppresses the shared greeting because its atomic Message stage owns both speech and the dialog. In text tests the fixed speech transcript carries a dedicated source marker, so preserve that exact output instead of guessing from unrelated fixed speech. """ greeting = shared_greeting.strip() if greeting: return greeting return next( (text.strip() for text in opening_speech_outputs if text.strip()), "", ) def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]: messages: list[dict[str, Any]] = [] latest_call_by_name: dict[str, str] = {} for index, turn in enumerate(context_turns): if turn.role in {"agent", "user"}: messages.append( { "role": "assistant" if turn.role == "agent" else "user", "content": turn.content, } ) continue call_id = turn.tool_call_id or latest_call_by_name.get(turn.tool_name or "") call_id = call_id or f"context_tool_{index}_{uuid4().hex[:8]}" if turn.role == "tool_call": latest_call_by_name[turn.tool_name or ""] = call_id messages.append( { "role": "assistant", "tool_calls": [ { "id": call_id, "type": "function", "function": { "name": turn.tool_name, "arguments": json.dumps( json.loads(turn.content), ensure_ascii=False ), }, } ], } ) else: result_payload = json.loads(turn.content) if turn.is_error: result_payload = ( {**result_payload, "status": "error"} if isinstance(result_payload, dict) else {"status": "error", "data": result_payload} ) messages.append( { "role": "tool", "tool_call_id": call_id, "content": json.dumps(result_payload, ensure_ascii=False), } ) return messages def _transcript_content(message: dict[str, Any]) -> str: """Keep evaluator transcripts readable without persisting image data URIs.""" content = message.get("content") if isinstance(content, str): return content.strip() if not isinstance(content, list): return "" text_parts = [ str(part.get("text") or "").strip() for part in content if isinstance(part, dict) and part.get("type") in {"text", "input_text"} and str(part.get("text") or "").strip() ] has_image = any( isinstance(part, dict) and part.get("type") in {"image_url", "input_image"} for part in content ) if has_image: text_parts.append("[用户提交了一张测试图片]") return "\n".join(text_parts).strip() async def _wait_for_quiescence( capture: _TextCaptureProcessor, runner_task: asyncio.Task[None], activity_marker: int, *, require_activity: bool = True, ) -> None: while True: if capture.error is not None: raise capture.error if runner_task.done(): await runner_task return has_activity = capture.window.activity_count > activity_marker settled = time.monotonic() - capture.window.last_activity_at >= IDLE_SETTLE_SECONDS if capture.window.idle and settled and (has_activity or not require_activity): return await asyncio.sleep(0.05) async def _wait_for_startup( worker: PipelineWorker, capture: _TextCaptureProcessor, runner_task: asyncio.Task[None], activity_marker: int, ) -> None: """Wait until startup-queued frames and any entry LLM reply are complete.""" boundary = asyncio.Event() await worker.queue_frame(_StartupBoundaryFrame(reached=boundary)) boundary_waiter = asyncio.create_task(boundary.wait()) done, _pending = await asyncio.wait( {boundary_waiter, runner_task}, timeout=WORKER_START_TIMEOUT_SECONDS, return_when=asyncio.FIRST_COMPLETED, ) if not done: boundary_waiter.cancel() raise TestExecutionError( code="STARTUP_DRAIN_TIMEOUT", message="开场阶段未能进入稳定状态", stage="pipeline", retryable=True, ) if boundary_waiter not in done: boundary_waiter.cancel() await asyncio.gather(boundary_waiter, return_exceptions=True) await _wait_for_quiescence( capture, runner_task, activity_marker, require_activity=False, ) class TextPipelineRunner: """Execute all fixed input turns in one isolated, transportless pipeline.""" async def run( self, cfg: AssistantConfig, definition: TestCaseDefinition, *, timeout_seconds: int, image_assets: dict[str, bytes] | None = None, on_turn_complete: TurnCompleteCallback | None = None, on_startup_complete: StartupCompleteCallback | None = None, ) -> TextCaseResult: try: async with asyncio.timeout(timeout_seconds): return await self._run( cfg, definition, image_assets=image_assets or {}, on_turn_complete=on_turn_complete, on_startup_complete=on_startup_complete, ) except TimeoutError as exc: raise TestExecutionError( code="PIPELINE_TIMEOUT", message=f"测试用例执行超过 {timeout_seconds} 秒", stage="pipeline", retryable=True, ) from exc async def _run( self, cfg: AssistantConfig, definition: TestCaseDefinition, *, image_assets: dict[str, bytes], on_turn_complete: TurnCompleteCallback | None, on_startup_complete: StartupCompleteCallback | None, ) -> TextCaseResult: if cfg.runtimeMode != "pipeline": raise TestExecutionError( code="UNSUPPORTED_RUNTIME_MODE", message="第一版文本测试只支持 Pipeline 运行模式", stage="pipeline", ) if cfg.type not in {"prompt", "workflow"}: raise TestExecutionError( code="UNSUPPORTED_ASSISTANT_TYPE", message=f"第一版文本测试暂不支持 {cfg.type} 类型助手", stage="pipeline", ) brain = build_brain(cfg) seed_messages = _context_messages(definition.context_turns) knowledge_config = cfg.knowledge_retrieval_config or {} knowledge_mode = str(knowledge_config.get("mode") or "automatic") def with_knowledge_hint(text: str) -> str: if cfg.type != "prompt" or not cfg.knowledge_base_id: return text hint = ( AUTOMATIC_KNOWLEDGE_HINT if knowledge_mode == "automatic" else ON_DEMAND_KNOWLEDGE_HINT ) return "\n\n".join(part for part in (text, hint) if part) system_prompt = with_knowledge_hint(brain.system_prompt(cfg)) context = LLMContext( messages=( seed_messages if cfg.type == "workflow" else [{"role": "system", "content": system_prompt}, *seed_messages] ) ) graph_settings = cfg.graph.get("settings") or {} default_resource = cfg.workflow_model_resources.get( str(graph_settings.get("defaultLlmResourceId") or "") ) llm_cfg = ( config_with_resource(cfg, default_resource) if cfg.type == "workflow" and default_resource is not None else cfg ) llm = brain.build_llm(llm_cfg, context) llm_services: dict[str, FrameProcessor] = {} current_llm_service: FrameProcessor = llm if cfg.type == "workflow": llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm) aggregators = LLMContextAggregatorPair(context) user_aggregator = aggregators.user() assistant_aggregator = aggregators.assistant() capture = _TextCaptureProcessor(context) knowledge = KnowledgeRetrievalProcessor( cfg.knowledge_base_id if knowledge_mode == "automatic" else None, top_n=int(knowledge_config.get("top_n", knowledge_config.get("topN", 5))), score_threshold=float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0) ) ), ) pipeline = Pipeline( [ user_aggregator, UserTurnRoutingProcessor(brain), knowledge, llm, capture, assistant_aggregator, ] ) worker = PipelineWorker( pipeline, params=PipelineParams(enable_metrics=False), enable_rtvi=False, enable_turn_tracking=False, idle_timeout_secs=None, ) runner = WorkerRunner(handle_sigint=False, check_dangling_tasks=False) worker_started = asyncio.Event() @worker.event_handler("on_pipeline_started") async def on_pipeline_started(_worker, _frame): worker_started.set() @worker.event_handler("on_pipeline_error") async def on_pipeline_error(_worker, frame): capture.fail(frame) executor_holder: dict[str, MockToolExecutor] = {} text_client_tools = TextClientToolPort() workflow_vision_scope: dict[str, Any] = {"enabled": False} def create_test_executor(store) -> MockToolExecutor: executor = MockToolExecutor(store) executor_holder["executor"] = executor return executor async def queue_call_end(reason: str) -> None: logger.debug(f"文本测试收到会话结束请求: {reason}") await worker.queue_frame( OutputTransportMessageUrgentFrame( message=call_end_trace_event( "ended", reason=reason, ) ) ) await worker.queue_frame(EndFrame(reason=reason)) async def record_call_end_state( state: str, interruptible: bool, ) -> None: await worker.queue_frame( OutputTransportMessageUrgentFrame( message=call_end_trace_event( state, reason=call_end.reason, interruptible=interruptible, ) ) ) call_end = CallEndCoordinator( queue_call_end, on_state_change=record_call_end_state, ) async def finish_fixed_speech_if_ending() -> None: if call_end.ending: await call_end.finish() capture.on_fixed_speech_complete = finish_fixed_speech_if_ending current_service = current_llm_service async def switch_services( llm_resource_id: str | None, _asr_resource_id: str | None, _tts_resource_id: str | None, ) -> None: nonlocal current_service target = ( llm_services.get(llm_resource_id) if llm_resource_id else current_llm_service ) if target is None: raise ValueError(f"Workflow LLM 资源未加载:{llm_resource_id}") if target is current_service: return await worker.queue_frame(ManuallySwitchServiceFrame(service=target)) current_service = target def set_system_prompt(text: str) -> None: messages = context.get_messages() text = with_knowledge_hint(text) if messages and messages[0].get("role") == "system": messages[0] = {"role": "system", "content": text} else: messages.insert(0, {"role": "system", "content": text}) knowledge_schema: FunctionSchema | None = None if ( cfg.type == "prompt" and cfg.knowledge_base_id and knowledge_mode == "on_demand" ): knowledge_schema = FunctionSchema( name="search_knowledge_base", description=( "在当前助手绑定的知识库中检索资料。" f"知识库：{cfg.knowledge_base_name}。" f"{cfg.knowledge_base_description}" ), properties={ "query": { "type": "string", "description": "完整问题或检索关键词", } }, required=["query"], ) async def search_bound_knowledge(params: FunctionCallParams) -> None: query = str(params.arguments.get("query") or "").strip() if not query: await params.result_callback( {"status": "error", "message": "检索问题为空"} ) return try: async with SessionLocal() as session: results = await search_knowledge( session, cfg.knowledge_base_id or "", query, top_k=int( knowledge_config.get( "top_n", knowledge_config.get("topN", 5), ) ), score_threshold=float( knowledge_config.get( "score_threshold", knowledge_config.get("scoreThreshold", 0.0), ) ), ) await params.result_callback( {"status": "ok", "results": results} ) except Exception as exc: # noqa: BLE001 - return tool errors to LLM logger.warning(f"文本测试知识库检索失败: {exc}") await params.result_callback( {"status": "error", "message": "知识库检索暂时不可用"} ) llm.register_function("search_knowledge_base", search_bound_knowledge) def set_tools(schemas=None) -> None: visible = list(schemas or []) if knowledge_schema is not None: visible.append(knowledge_schema) if visible: context.set_tools(ToolsSchema(standard_tools=visible)) else: context.set_tools() await brain.setup( cfg, BrainRuntime( context=context, llm=llm, queue_frame=worker.queue_frame, set_system_prompt=set_system_prompt, set_tools=set_tools, call_end=call_end, session_id=f"test_{uuid4().hex}", client_tools=text_client_tools, worker=worker, context_aggregator=aggregators, transport=None, switch_services=switch_services, set_knowledge_scope=knowledge.set_scope, set_vision_scope=lambda scope: workflow_vision_scope.update(scope), vision_function=None, set_input_enabled=lambda _enabled: None, apply_turn_config=lambda _enabled, _config: asyncio.sleep(0), flow_global_functions=[], tool_executor_factory=create_test_executor, ), ) executor = executor_holder["executor"] # A transportless text run cannot synthesize audio, but the configured # greeting has already completed before scripted inputs begin. Mirror # production's provider-safe context fact so the model does not repeat it. greeting = str(await brain.greeting(cfg) or "").strip() if greeting and brain.spec.owns_context: brain.prepare_greeting_context(greeting, context) @assistant_aggregator.event_handler("on_assistant_turn_started") async def on_assistant_turn_started(_aggregator): capture.window.active_assistant += 1 capture.window.touch() await brain.on_assistant_text_start(uuid4().hex) @assistant_aggregator.event_handler("on_assistant_turn_stopped") async def on_assistant_turn_stopped(_aggregator, message): content = str(message.content or "").strip() capture.window.active_assistant = max( 0, capture.window.active_assistant - 1 ) if content: capture.outputs.append(content) capture.window.touch() await brain.on_assistant_text_end( uuid4().hex, content, bool(getattr(message, "interrupted", False)), ) if call_end.ending: # Text tests have no audio transport. A fully aggregated # response is therefore the equivalent of playback ending. await call_end.finish() all_tool_behaviors = [ behavior for turn in definition.turns for behavior in turn.behaviors if behavior.type == "tool_call" ] executor.set_turn_behaviors(all_tool_behaviors) runner_task: asyncio.Task[None] | None = None try: startup_runtime_record_marker = len(capture.runtime_records) await brain.run_preflight() if executor.unmocked_calls: raise TestExecutionError( code="UNMOCKED_TOOL_CALL", message="启动阶段调用了未配置 Mock 的工具: " + "、".join(executor.unmocked_calls), stage="tool", ) await runner.add_workers(worker) runner_task = asyncio.create_task( runner.run(), name=f"text-test-{uuid4().hex[:12]}" ) await asyncio.wait_for( worker_started.wait(), timeout=WORKER_START_TIMEOUT_SECONDS ) marker = capture.window.activity_count await brain.on_connected(greeting_pending=False) await brain.on_client_ready() await _wait_for_startup( worker, capture, runner_task, marker, ) startup_greeting = _resolve_startup_greeting( greeting, capture.opening_speech_outputs, ) startup = TextStartupResult( greeting=startup_greeting, assistant_outputs=list(capture.outputs), auto_confirmed_messages=list( text_client_tools.auto_confirmed_messages ), runtime_records=list( capture.runtime_records[startup_runtime_record_marker:] ), ) if on_startup_complete is not None: await on_startup_complete(startup) # Startup messages initialize the session but are not a scripted # user turn result in the fixed-input editor contract. capture.outputs.clear() capture.tool_calls.clear() results: list[RawTurnResult] = [] for index, turn in enumerate(definition.turns): if worker.has_finished(): raise TestExecutionError( code="PIPELINE_ENDED_EARLY", message=f"Pipeline 在第 {index + 1} 轮输入前已经结束", stage="pipeline", retryable=False, ) result = await self._run_turn( cfg, worker, runner_task, capture, executor, turn, index, image_assets=image_assets, workflow_vision_scope=workflow_vision_scope, ) results.append(result) if on_turn_complete is not None: await on_turn_complete(result) transcript = [] for message in context.get_messages(): if message.get("role") not in {"user", "assistant"}: continue content = _transcript_content(message) if content: transcript.append( { "role": str(message.get("role") or ""), "content": content, } ) return TextCaseResult( turns=results, transcript=transcript, startup=startup, ) except TestExecutionError: raise except asyncio.CancelledError: raise except Exception as exc: raise TestExecutionError( code="PIPELINE_EXECUTION_FAILED", message=str(exc) or type(exc).__name__, stage="model", retryable=True, ) from exc finally: if not worker.has_finished(): await worker.cancel(reason=INTERNAL_CANCEL_REASON) if runner_task is not None: await asyncio.gather(runner_task, return_exceptions=True) async def _run_turn( self, cfg: AssistantConfig, worker: PipelineWorker, runner_task: asyncio.Task[None], capture: _TextCaptureProcessor, executor: MockToolExecutor, turn: FixedInputTurn, index: int, *, image_assets: dict[str, bytes], workflow_vision_scope: dict[str, Any], ) -> RawTurnResult: tool_behaviors = [ behavior for behavior in turn.behaviors if behavior.type == "tool_call" ] executor.set_turn_behaviors(tool_behaviors) activity_marker = capture.window.activity_count output_marker = len(capture.outputs) tool_marker = len(capture.tool_calls) runtime_record_marker = len(capture.runtime_records) user_message: dict[str, Any] = { "role": "user", "content": turn.user_input, } if turn.image is not None: image_bytes = image_assets.get(turn.image.asset_id) if image_bytes is None: raise TestExecutionError( code="TEST_IMAGE_MISSING", message=f"测试图片不存在: {turn.image.asset_id}", stage="pipeline", retryable=False, ) try: user_message = await build_image_user_message( cfg, image_bytes, turn.user_input, workflow_scope=( workflow_vision_scope if cfg.type == "workflow" else None ), ) except ValueError as exc: raise TestExecutionError( code="TEST_IMAGE_UNSUPPORTED", message=str(exc), stage="model", retryable=False, ) from exc await worker.queue_frame( LLMMessagesAppendFrame(messages=[user_message], run_llm=True) ) await _wait_for_quiescence(capture, runner_task, activity_marker) if executor.unmocked_calls: raise TestExecutionError( code="UNMOCKED_TOOL_CALL", message="本轮调用了未配置 Mock 的工具: " + "、".join(executor.unmocked_calls), stage="tool", ) return RawTurnResult( id=turn.id, index=index, user_input=turn.user_input, image=turn.image, assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(), tool_calls=list(capture.tool_calls[tool_marker:]), runtime_records=list( capture.runtime_records[runtime_record_marker:] ), )