Xin Wang
7560f9a525
- Introduced dynamic_variables field in BatchRunCreate schema to support flexible input types. - Added BatchStartupResult schema to encapsulate startup state, including greeting and auto-confirmed messages. - Updated BatchRunCaseOut schema to include startup results, enhancing the structure for test run outputs. - Enhanced test run orchestration to prepare and validate dynamic configurations, ensuring isolated variable management across test cases. - Implemented changes in the frontend to accommodate new data structures for improved test case execution and result tracking.
799 lines
29 KiB
Python
799 lines
29 KiB
Python
"""Transportless Pipecat runner for one persisted fixed-text test case."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Awaitable, Callable
|
|
from uuid import uuid4
|
|
|
|
from loguru import logger
|
|
from models import AssistantConfig
|
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|
from pipecat.frames.frames import (
|
|
BotStoppedSpeakingFrame,
|
|
CancelFrame,
|
|
DataFrame,
|
|
EndFrame,
|
|
ErrorFrame,
|
|
FunctionCallInProgressFrame,
|
|
FunctionCallResultFrame,
|
|
LLMContextFrame,
|
|
LLMFullResponseEndFrame,
|
|
LLMFullResponseStartFrame,
|
|
LLMMessagesAppendFrame,
|
|
ManuallySwitchServiceFrame,
|
|
TTSSpeakFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
LLMContextAggregatorPair,
|
|
)
|
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|
from pipecat.services.llm_service import FunctionCallParams
|
|
from pipecat.workers.runner import WorkerRunner
|
|
|
|
from db.session import SessionLocal
|
|
from services.brains import BrainRuntime, build_brain
|
|
from services.knowledge import search as search_knowledge
|
|
from services.pipecat.call_lifecycle import (
|
|
CallEndCoordinator,
|
|
FixedSpeechPlaybackMarkerFrame,
|
|
)
|
|
from services.pipecat.processors import (
|
|
KnowledgeRetrievalProcessor,
|
|
UserTurnRoutingProcessor,
|
|
)
|
|
from services.pipecat.service_factory import config_with_resource
|
|
from services.pipecat.workflow_services import build_workflow_llm_switcher
|
|
from services.test_runs.errors import TestExecutionError
|
|
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
|
|
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition
|
|
|
|
|
|
IDLE_SETTLE_SECONDS = 0.2
|
|
WORKER_START_TIMEOUT_SECONDS = 5.0
|
|
INTERNAL_CANCEL_REASON = "text_test_case_complete"
|
|
AUTOMATIC_KNOWLEDGE_HINT = (
|
|
"你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
|
|
"回答资料事实时只根据检索内容，资料不足要明确说明。"
|
|
)
|
|
ON_DEMAND_KNOWLEDGE_HINT = (
|
|
"你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，"
|
|
"先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，"
|
|
"资料不足要明确说明。"
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class RawToolCall:
|
|
id: str
|
|
function_name: str
|
|
arguments: Any
|
|
result: Any = None
|
|
outcome: str = "error"
|
|
duration_ms: int = 0
|
|
_started_at: float = field(default_factory=time.monotonic, repr=False)
|
|
|
|
def as_result(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id,
|
|
"functionName": self.function_name,
|
|
"argumentsJson": json.dumps(
|
|
self.arguments, ensure_ascii=False, indent=2, default=str
|
|
),
|
|
"resultJson": json.dumps(
|
|
self.result, ensure_ascii=False, indent=2, default=str
|
|
),
|
|
"outcome": self.outcome,
|
|
"durationMs": self.duration_ms,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class RawTurnResult:
|
|
id: str
|
|
index: int
|
|
user_input: str
|
|
assistant_reply: str
|
|
tool_calls: list[RawToolCall]
|
|
|
|
|
|
@dataclass
|
|
class TextStartupResult:
|
|
"""Headless startup choices applied before the scripted user turns."""
|
|
|
|
greeting: str = ""
|
|
assistant_outputs: list[str] = field(default_factory=list)
|
|
auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class TextCaseResult:
|
|
turns: list[RawTurnResult]
|
|
transcript: list[dict[str, str]]
|
|
startup: TextStartupResult = field(default_factory=TextStartupResult)
|
|
|
|
|
|
TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]]
|
|
StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]]
|
|
|
|
|
|
@dataclass
|
|
class _StartupBoundaryFrame(DataFrame):
|
|
"""Ordered marker queued behind every frame emitted by startup."""
|
|
|
|
reached: asyncio.Event
|
|
|
|
|
|
@dataclass
|
|
class _ResponseWindow:
|
|
active_llm: int = 0
|
|
active_assistant: int = 0
|
|
pending_tools: set[str] = field(default_factory=set)
|
|
activity_count: int = 0
|
|
last_activity_at: float = field(default_factory=time.monotonic)
|
|
|
|
def touch(self) -> None:
|
|
self.activity_count += 1
|
|
self.last_activity_at = time.monotonic()
|
|
|
|
@property
|
|
def idle(self) -> bool:
|
|
return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools
|
|
|
|
|
|
class _TextCaptureProcessor(FrameProcessor):
|
|
"""Capture assistant/tool output and expose a reliable response boundary."""
|
|
|
|
def __init__(self, context: LLMContext):
|
|
super().__init__()
|
|
self.context = context
|
|
self.window = _ResponseWindow()
|
|
self.outputs: list[str] = []
|
|
self.tool_calls: list[RawToolCall] = []
|
|
self._tool_by_id: dict[str, RawToolCall] = {}
|
|
self.error: TestExecutionError | None = None
|
|
self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None
|
|
|
|
def fail(self, frame: ErrorFrame) -> None:
|
|
"""Release every wait even when Pipecat reports the error upstream."""
|
|
|
|
message = frame.error or "Pipeline 执行失败"
|
|
unmocked_tool = "UNMOCKED_TOOL_CALL" in message
|
|
self.error = TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR",
|
|
message=message,
|
|
stage=(
|
|
"tool"
|
|
if unmocked_tool
|
|
else "model"
|
|
if frame.processor
|
|
else "pipeline"
|
|
),
|
|
retryable=not unmocked_tool,
|
|
)
|
|
self.window.pending_tools.clear()
|
|
self.window.touch()
|
|
|
|
async def process_frame(self, frame, direction: FrameDirection):
|
|
await super().process_frame(frame, direction)
|
|
|
|
if isinstance(frame, TTSSpeakFrame):
|
|
text = frame.text.strip()
|
|
if text:
|
|
self.outputs.append(text)
|
|
if frame.append_to_context:
|
|
self.context.add_message({"role": "assistant", "content": text})
|
|
self.window.touch()
|
|
if self.on_fixed_speech_complete is not None:
|
|
await self.on_fixed_speech_complete()
|
|
return
|
|
|
|
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
|
|
await frame.completion.mark_played()
|
|
self.window.touch()
|
|
return
|
|
|
|
if isinstance(frame, LLMFullResponseStartFrame):
|
|
self.window.active_llm += 1
|
|
self.window.touch()
|
|
elif isinstance(frame, LLMFullResponseEndFrame):
|
|
self.window.active_llm = max(0, self.window.active_llm - 1)
|
|
self.window.touch()
|
|
await self.push_frame(frame, direction)
|
|
# There is no TTS/output transport. This releases provider/tool
|
|
# follow-up logic that normally waits for a speaking boundary.
|
|
await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
|
|
return
|
|
elif isinstance(frame, FunctionCallInProgressFrame):
|
|
call = RawToolCall(
|
|
id=frame.tool_call_id,
|
|
function_name=frame.function_name,
|
|
arguments=frame.arguments or {},
|
|
)
|
|
self.tool_calls.append(call)
|
|
self._tool_by_id[call.id] = call
|
|
self.window.pending_tools.add(call.id)
|
|
self.window.touch()
|
|
elif isinstance(frame, FunctionCallResultFrame):
|
|
call = self._tool_by_id.get(frame.tool_call_id)
|
|
if call is None:
|
|
call = RawToolCall(
|
|
id=frame.tool_call_id,
|
|
function_name=frame.function_name,
|
|
arguments=frame.arguments or {},
|
|
)
|
|
self.tool_calls.append(call)
|
|
self._tool_by_id[call.id] = call
|
|
call.result = frame.result
|
|
call.duration_ms = max(0, round((time.monotonic() - call._started_at) * 1000))
|
|
status = frame.result.get("status") if isinstance(frame.result, dict) else None
|
|
call.outcome = "error" if status in {"error", "failed", "timeout"} else "success"
|
|
self.window.pending_tools.discard(call.id)
|
|
self.window.touch()
|
|
elif isinstance(frame, ErrorFrame):
|
|
self.fail(frame)
|
|
elif isinstance(frame, _StartupBoundaryFrame):
|
|
frame.reached.set()
|
|
self.window.touch()
|
|
elif isinstance(frame, (EndFrame, CancelFrame)):
|
|
self.window.touch()
|
|
|
|
await self.push_frame(frame, direction)
|
|
|
|
|
|
def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
|
|
messages: list[dict[str, Any]] = []
|
|
latest_call_by_name: dict[str, str] = {}
|
|
for index, turn in enumerate(context_turns):
|
|
if turn.role in {"agent", "user"}:
|
|
messages.append(
|
|
{
|
|
"role": "assistant" if turn.role == "agent" else "user",
|
|
"content": turn.content,
|
|
}
|
|
)
|
|
continue
|
|
|
|
call_id = turn.tool_call_id or latest_call_by_name.get(turn.tool_name or "")
|
|
call_id = call_id or f"context_tool_{index}_{uuid4().hex[:8]}"
|
|
if turn.role == "tool_call":
|
|
latest_call_by_name[turn.tool_name or ""] = call_id
|
|
messages.append(
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": call_id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": turn.tool_name,
|
|
"arguments": json.dumps(
|
|
json.loads(turn.content), ensure_ascii=False
|
|
),
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
else:
|
|
result_payload = json.loads(turn.content)
|
|
if turn.is_error:
|
|
result_payload = (
|
|
{**result_payload, "status": "error"}
|
|
if isinstance(result_payload, dict)
|
|
else {"status": "error", "data": result_payload}
|
|
)
|
|
messages.append(
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": call_id,
|
|
"content": json.dumps(result_payload, ensure_ascii=False),
|
|
}
|
|
)
|
|
return messages
|
|
|
|
|
|
async def _wait_for_quiescence(
|
|
capture: _TextCaptureProcessor,
|
|
runner_task: asyncio.Task[None],
|
|
activity_marker: int,
|
|
*,
|
|
require_activity: bool = True,
|
|
) -> None:
|
|
while True:
|
|
if capture.error is not None:
|
|
raise capture.error
|
|
if runner_task.done():
|
|
await runner_task
|
|
return
|
|
has_activity = capture.window.activity_count > activity_marker
|
|
settled = time.monotonic() - capture.window.last_activity_at >= IDLE_SETTLE_SECONDS
|
|
if capture.window.idle and settled and (has_activity or not require_activity):
|
|
return
|
|
await asyncio.sleep(0.05)
|
|
|
|
|
|
async def _wait_for_startup(
|
|
worker: PipelineWorker,
|
|
capture: _TextCaptureProcessor,
|
|
runner_task: asyncio.Task[None],
|
|
activity_marker: int,
|
|
) -> None:
|
|
"""Wait until startup-queued frames and any entry LLM reply are complete."""
|
|
|
|
boundary = asyncio.Event()
|
|
await worker.queue_frame(_StartupBoundaryFrame(reached=boundary))
|
|
boundary_waiter = asyncio.create_task(boundary.wait())
|
|
done, _pending = await asyncio.wait(
|
|
{boundary_waiter, runner_task},
|
|
timeout=WORKER_START_TIMEOUT_SECONDS,
|
|
return_when=asyncio.FIRST_COMPLETED,
|
|
)
|
|
if not done:
|
|
boundary_waiter.cancel()
|
|
raise TestExecutionError(
|
|
code="STARTUP_DRAIN_TIMEOUT",
|
|
message="开场阶段未能进入稳定状态",
|
|
stage="pipeline",
|
|
retryable=True,
|
|
)
|
|
if boundary_waiter not in done:
|
|
boundary_waiter.cancel()
|
|
await asyncio.gather(boundary_waiter, return_exceptions=True)
|
|
await _wait_for_quiescence(
|
|
capture,
|
|
runner_task,
|
|
activity_marker,
|
|
require_activity=False,
|
|
)
|
|
|
|
|
|
class TextPipelineRunner:
|
|
"""Execute all fixed input turns in one isolated, transportless pipeline."""
|
|
|
|
async def run(
|
|
self,
|
|
cfg: AssistantConfig,
|
|
definition: TestCaseDefinition,
|
|
*,
|
|
timeout_seconds: int,
|
|
on_turn_complete: TurnCompleteCallback | None = None,
|
|
on_startup_complete: StartupCompleteCallback | None = None,
|
|
) -> TextCaseResult:
|
|
try:
|
|
async with asyncio.timeout(timeout_seconds):
|
|
return await self._run(
|
|
cfg,
|
|
definition,
|
|
on_turn_complete=on_turn_complete,
|
|
on_startup_complete=on_startup_complete,
|
|
)
|
|
except TimeoutError as exc:
|
|
raise TestExecutionError(
|
|
code="PIPELINE_TIMEOUT",
|
|
message=f"测试用例执行超过 {timeout_seconds} 秒",
|
|
stage="pipeline",
|
|
retryable=True,
|
|
) from exc
|
|
|
|
async def _run(
|
|
self,
|
|
cfg: AssistantConfig,
|
|
definition: TestCaseDefinition,
|
|
*,
|
|
on_turn_complete: TurnCompleteCallback | None,
|
|
on_startup_complete: StartupCompleteCallback | None,
|
|
) -> TextCaseResult:
|
|
if cfg.runtimeMode != "pipeline":
|
|
raise TestExecutionError(
|
|
code="UNSUPPORTED_RUNTIME_MODE",
|
|
message="第一版文本测试只支持 Pipeline 运行模式",
|
|
stage="pipeline",
|
|
)
|
|
if cfg.type not in {"prompt", "workflow"}:
|
|
raise TestExecutionError(
|
|
code="UNSUPPORTED_ASSISTANT_TYPE",
|
|
message=f"第一版文本测试暂不支持 {cfg.type} 类型助手",
|
|
stage="pipeline",
|
|
)
|
|
|
|
brain = build_brain(cfg)
|
|
seed_messages = _context_messages(definition.context_turns)
|
|
knowledge_config = cfg.knowledge_retrieval_config or {}
|
|
knowledge_mode = str(knowledge_config.get("mode") or "automatic")
|
|
|
|
def with_knowledge_hint(text: str) -> str:
|
|
if cfg.type != "prompt" or not cfg.knowledge_base_id:
|
|
return text
|
|
hint = (
|
|
AUTOMATIC_KNOWLEDGE_HINT
|
|
if knowledge_mode == "automatic"
|
|
else ON_DEMAND_KNOWLEDGE_HINT
|
|
)
|
|
return "\n\n".join(part for part in (text, hint) if part)
|
|
|
|
system_prompt = with_knowledge_hint(brain.system_prompt(cfg))
|
|
context = LLMContext(
|
|
messages=(
|
|
seed_messages
|
|
if cfg.type == "workflow"
|
|
else [{"role": "system", "content": system_prompt}, *seed_messages]
|
|
)
|
|
)
|
|
|
|
graph_settings = cfg.graph.get("settings") or {}
|
|
default_resource = cfg.workflow_model_resources.get(
|
|
str(graph_settings.get("defaultLlmResourceId") or "")
|
|
)
|
|
llm_cfg = (
|
|
config_with_resource(cfg, default_resource)
|
|
if cfg.type == "workflow" and default_resource is not None
|
|
else cfg
|
|
)
|
|
llm = brain.build_llm(llm_cfg, context)
|
|
llm_services: dict[str, FrameProcessor] = {}
|
|
current_llm_service: FrameProcessor = llm
|
|
if cfg.type == "workflow":
|
|
llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm)
|
|
|
|
aggregators = LLMContextAggregatorPair(context)
|
|
user_aggregator = aggregators.user()
|
|
assistant_aggregator = aggregators.assistant()
|
|
capture = _TextCaptureProcessor(context)
|
|
knowledge = KnowledgeRetrievalProcessor(
|
|
cfg.knowledge_base_id if knowledge_mode == "automatic" else None,
|
|
top_n=int(knowledge_config.get("top_n", knowledge_config.get("topN", 5))),
|
|
score_threshold=float(
|
|
knowledge_config.get(
|
|
"score_threshold", knowledge_config.get("scoreThreshold", 0.0)
|
|
)
|
|
),
|
|
)
|
|
pipeline = Pipeline(
|
|
[
|
|
user_aggregator,
|
|
UserTurnRoutingProcessor(brain),
|
|
knowledge,
|
|
llm,
|
|
capture,
|
|
assistant_aggregator,
|
|
]
|
|
)
|
|
worker = PipelineWorker(
|
|
pipeline,
|
|
params=PipelineParams(enable_metrics=False),
|
|
enable_rtvi=False,
|
|
enable_turn_tracking=False,
|
|
idle_timeout_secs=None,
|
|
)
|
|
runner = WorkerRunner(handle_sigint=False, check_dangling_tasks=False)
|
|
worker_started = asyncio.Event()
|
|
|
|
@worker.event_handler("on_pipeline_started")
|
|
async def on_pipeline_started(_worker, _frame):
|
|
worker_started.set()
|
|
|
|
@worker.event_handler("on_pipeline_error")
|
|
async def on_pipeline_error(_worker, frame):
|
|
capture.fail(frame)
|
|
|
|
executor_holder: dict[str, MockToolExecutor] = {}
|
|
text_client_tools = TextClientToolPort()
|
|
|
|
def create_test_executor(store) -> MockToolExecutor:
|
|
executor = MockToolExecutor(store)
|
|
executor_holder["executor"] = executor
|
|
return executor
|
|
|
|
async def queue_call_end(reason: str) -> None:
|
|
logger.debug(f"文本测试收到会话结束请求: {reason}")
|
|
await worker.queue_frame(EndFrame(reason=reason))
|
|
|
|
call_end = CallEndCoordinator(queue_call_end)
|
|
|
|
async def finish_fixed_speech_if_ending() -> None:
|
|
if call_end.ending:
|
|
await call_end.finish()
|
|
|
|
capture.on_fixed_speech_complete = finish_fixed_speech_if_ending
|
|
current_service = current_llm_service
|
|
|
|
async def switch_services(
|
|
llm_resource_id: str | None,
|
|
_asr_resource_id: str | None,
|
|
_tts_resource_id: str | None,
|
|
) -> None:
|
|
nonlocal current_service
|
|
target = (
|
|
llm_services.get(llm_resource_id)
|
|
if llm_resource_id
|
|
else current_llm_service
|
|
)
|
|
if target is None:
|
|
raise ValueError(f"Workflow LLM 资源未加载:{llm_resource_id}")
|
|
if target is current_service:
|
|
return
|
|
await worker.queue_frame(ManuallySwitchServiceFrame(service=target))
|
|
current_service = target
|
|
|
|
def set_system_prompt(text: str) -> None:
|
|
messages = context.get_messages()
|
|
text = with_knowledge_hint(text)
|
|
if messages and messages[0].get("role") == "system":
|
|
messages[0] = {"role": "system", "content": text}
|
|
else:
|
|
messages.insert(0, {"role": "system", "content": text})
|
|
|
|
knowledge_schema: FunctionSchema | None = None
|
|
if (
|
|
cfg.type == "prompt"
|
|
and cfg.knowledge_base_id
|
|
and knowledge_mode == "on_demand"
|
|
):
|
|
knowledge_schema = FunctionSchema(
|
|
name="search_knowledge_base",
|
|
description=(
|
|
"在当前助手绑定的知识库中检索资料。"
|
|
f"知识库：{cfg.knowledge_base_name}。"
|
|
f"{cfg.knowledge_base_description}"
|
|
),
|
|
properties={
|
|
"query": {
|
|
"type": "string",
|
|
"description": "完整问题或检索关键词",
|
|
}
|
|
},
|
|
required=["query"],
|
|
)
|
|
|
|
async def search_bound_knowledge(params: FunctionCallParams) -> None:
|
|
query = str(params.arguments.get("query") or "").strip()
|
|
if not query:
|
|
await params.result_callback(
|
|
{"status": "error", "message": "检索问题为空"}
|
|
)
|
|
return
|
|
try:
|
|
async with SessionLocal() as session:
|
|
results = await search_knowledge(
|
|
session,
|
|
cfg.knowledge_base_id or "",
|
|
query,
|
|
top_k=int(
|
|
knowledge_config.get(
|
|
"top_n",
|
|
knowledge_config.get("topN", 5),
|
|
)
|
|
),
|
|
score_threshold=float(
|
|
knowledge_config.get(
|
|
"score_threshold",
|
|
knowledge_config.get("scoreThreshold", 0.0),
|
|
)
|
|
),
|
|
)
|
|
await params.result_callback(
|
|
{"status": "ok", "results": results}
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - return tool errors to LLM
|
|
logger.warning(f"文本测试知识库检索失败: {exc}")
|
|
await params.result_callback(
|
|
{"status": "error", "message": "知识库检索暂时不可用"}
|
|
)
|
|
|
|
llm.register_function("search_knowledge_base", search_bound_knowledge)
|
|
|
|
def set_tools(schemas=None) -> None:
|
|
visible = list(schemas or [])
|
|
if knowledge_schema is not None:
|
|
visible.append(knowledge_schema)
|
|
if visible:
|
|
context.set_tools(ToolsSchema(standard_tools=visible))
|
|
else:
|
|
context.set_tools()
|
|
|
|
await brain.setup(
|
|
cfg,
|
|
BrainRuntime(
|
|
context=context,
|
|
llm=llm,
|
|
queue_frame=worker.queue_frame,
|
|
set_system_prompt=set_system_prompt,
|
|
set_tools=set_tools,
|
|
call_end=call_end,
|
|
session_id=f"test_{uuid4().hex}",
|
|
client_tools=text_client_tools,
|
|
worker=worker,
|
|
context_aggregator=aggregators,
|
|
transport=None,
|
|
switch_services=switch_services,
|
|
set_knowledge_scope=knowledge.set_scope,
|
|
set_vision_scope=lambda _scope: None,
|
|
vision_function=None,
|
|
set_input_enabled=lambda _enabled: None,
|
|
apply_turn_config=lambda _enabled, _config: asyncio.sleep(0),
|
|
flow_global_functions=[],
|
|
tool_executor_factory=create_test_executor,
|
|
),
|
|
)
|
|
executor = executor_holder["executor"]
|
|
|
|
# A transportless text run cannot synthesize audio, but the configured
|
|
# greeting has already completed before scripted inputs begin. Mirror
|
|
# production's provider-safe context fact so the model does not repeat it.
|
|
greeting = str(await brain.greeting(cfg) or "").strip()
|
|
if greeting and brain.spec.owns_context:
|
|
brain.prepare_greeting_context(greeting, context)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_turn_started")
|
|
async def on_assistant_turn_started(_aggregator):
|
|
capture.window.active_assistant += 1
|
|
capture.window.touch()
|
|
await brain.on_assistant_text_start(uuid4().hex)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
|
async def on_assistant_turn_stopped(_aggregator, message):
|
|
content = str(message.content or "").strip()
|
|
capture.window.active_assistant = max(
|
|
0, capture.window.active_assistant - 1
|
|
)
|
|
if content:
|
|
capture.outputs.append(content)
|
|
capture.window.touch()
|
|
await brain.on_assistant_text_end(
|
|
uuid4().hex,
|
|
content,
|
|
bool(getattr(message, "interrupted", False)),
|
|
)
|
|
if call_end.ending:
|
|
# Text tests have no audio transport. A fully aggregated
|
|
# response is therefore the equivalent of playback ending.
|
|
await call_end.finish()
|
|
|
|
all_tool_behaviors = [
|
|
behavior
|
|
for turn in definition.turns
|
|
for behavior in turn.behaviors
|
|
if behavior.type == "tool_call"
|
|
]
|
|
executor.set_turn_behaviors(all_tool_behaviors)
|
|
|
|
runner_task: asyncio.Task[None] | None = None
|
|
try:
|
|
await brain.run_preflight()
|
|
if executor.unmocked_calls:
|
|
raise TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL",
|
|
message="启动阶段调用了未配置 Mock 的工具: "
|
|
+ "、".join(executor.unmocked_calls),
|
|
stage="tool",
|
|
)
|
|
|
|
await runner.add_workers(worker)
|
|
runner_task = asyncio.create_task(
|
|
runner.run(), name=f"text-test-{uuid4().hex[:12]}"
|
|
)
|
|
await asyncio.wait_for(
|
|
worker_started.wait(), timeout=WORKER_START_TIMEOUT_SECONDS
|
|
)
|
|
|
|
marker = capture.window.activity_count
|
|
await brain.on_connected(greeting_pending=False)
|
|
await brain.on_client_ready()
|
|
await _wait_for_startup(
|
|
worker,
|
|
capture,
|
|
runner_task,
|
|
marker,
|
|
)
|
|
startup = TextStartupResult(
|
|
greeting=greeting,
|
|
assistant_outputs=list(capture.outputs),
|
|
auto_confirmed_messages=list(
|
|
text_client_tools.auto_confirmed_messages
|
|
),
|
|
)
|
|
if on_startup_complete is not None:
|
|
await on_startup_complete(startup)
|
|
# Startup messages initialize the session but are not a scripted
|
|
# user turn result in the fixed-input editor contract.
|
|
capture.outputs.clear()
|
|
capture.tool_calls.clear()
|
|
|
|
results: list[RawTurnResult] = []
|
|
for index, turn in enumerate(definition.turns):
|
|
if worker.has_finished():
|
|
raise TestExecutionError(
|
|
code="PIPELINE_ENDED_EARLY",
|
|
message=f"Pipeline 在第 {index + 1} 轮输入前已经结束",
|
|
stage="pipeline",
|
|
retryable=False,
|
|
)
|
|
result = await self._run_turn(
|
|
worker,
|
|
runner_task,
|
|
capture,
|
|
executor,
|
|
turn,
|
|
index,
|
|
)
|
|
results.append(result)
|
|
if on_turn_complete is not None:
|
|
await on_turn_complete(result)
|
|
|
|
transcript = [
|
|
{
|
|
"role": str(message.get("role") or ""),
|
|
"content": str(message.get("content") or ""),
|
|
}
|
|
for message in context.get_messages()
|
|
if message.get("role") in {"user", "assistant"}
|
|
and str(message.get("content") or "").strip()
|
|
]
|
|
return TextCaseResult(
|
|
turns=results,
|
|
transcript=transcript,
|
|
startup=startup,
|
|
)
|
|
except TestExecutionError:
|
|
raise
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
raise TestExecutionError(
|
|
code="PIPELINE_EXECUTION_FAILED",
|
|
message=str(exc) or type(exc).__name__,
|
|
stage="model",
|
|
retryable=True,
|
|
) from exc
|
|
finally:
|
|
if not worker.has_finished():
|
|
await worker.cancel(reason=INTERNAL_CANCEL_REASON)
|
|
if runner_task is not None:
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
async def _run_turn(
|
|
self,
|
|
worker: PipelineWorker,
|
|
runner_task: asyncio.Task[None],
|
|
capture: _TextCaptureProcessor,
|
|
executor: MockToolExecutor,
|
|
turn: FixedInputTurn,
|
|
index: int,
|
|
) -> RawTurnResult:
|
|
tool_behaviors = [
|
|
behavior for behavior in turn.behaviors if behavior.type == "tool_call"
|
|
]
|
|
executor.set_turn_behaviors(tool_behaviors)
|
|
activity_marker = capture.window.activity_count
|
|
output_marker = len(capture.outputs)
|
|
tool_marker = len(capture.tool_calls)
|
|
await worker.queue_frame(
|
|
LLMMessagesAppendFrame(
|
|
messages=[{"role": "user", "content": turn.user_input}],
|
|
run_llm=True,
|
|
)
|
|
)
|
|
await _wait_for_quiescence(capture, runner_task, activity_marker)
|
|
if executor.unmocked_calls:
|
|
raise TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL",
|
|
message="本轮调用了未配置 Mock 的工具: "
|
|
+ "、".join(executor.unmocked_calls),
|
|
stage="tool",
|
|
)
|
|
return RawTurnResult(
|
|
id=turn.id,
|
|
index=index,
|
|
user_input=turn.user_input,
|
|
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
|
|
tool_calls=list(capture.tool_calls[tool_marker:]),
|
|
)
|