963 lines
35 KiB
Python
963 lines
35 KiB
Python
"""Transportless Pipecat runner for one persisted fixed-text test case."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Awaitable, Callable
|
|
from uuid import uuid4
|
|
|
|
from loguru import logger
|
|
from models import AssistantConfig
|
|
from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|
from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|
from pipecat.frames.frames import (
|
|
BotStoppedSpeakingFrame,
|
|
CancelFrame,
|
|
DataFrame,
|
|
EndFrame,
|
|
ErrorFrame,
|
|
FunctionCallInProgressFrame,
|
|
FunctionCallResultFrame,
|
|
LLMContextFrame,
|
|
LLMFullResponseEndFrame,
|
|
LLMFullResponseStartFrame,
|
|
LLMMessagesAppendFrame,
|
|
ManuallySwitchServiceFrame,
|
|
OutputTransportMessageUrgentFrame,
|
|
TTSSpeakFrame,
|
|
)
|
|
from pipecat.pipeline.pipeline import Pipeline
|
|
from pipecat.pipeline.worker import PipelineParams, PipelineWorker
|
|
from pipecat.processors.aggregators.llm_context import LLMContext
|
|
from pipecat.processors.aggregators.llm_response_universal import (
|
|
LLMContextAggregatorPair,
|
|
)
|
|
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|
from pipecat.services.llm_service import FunctionCallParams
|
|
from pipecat.workers.runner import WorkerRunner
|
|
|
|
from db.session import SessionLocal
|
|
from services.brains import BrainRuntime, build_brain
|
|
from services.knowledge import search as search_knowledge
|
|
from services.pipecat.call_lifecycle import (
|
|
CallEndCoordinator,
|
|
FixedSpeechPlaybackMarkerFrame,
|
|
call_end_trace_event,
|
|
)
|
|
from services.pipecat.processors import (
|
|
KnowledgeRetrievalProcessor,
|
|
UserTurnRoutingProcessor,
|
|
)
|
|
from services.pipecat.service_factory import config_with_resource
|
|
from services.pipecat.workflow_services import build_workflow_llm_switcher
|
|
from services.test_runs.errors import TestExecutionError
|
|
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
|
|
from services.vision import build_image_user_message
|
|
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef
|
|
|
|
|
|
IDLE_SETTLE_SECONDS = 0.2
|
|
WORKER_START_TIMEOUT_SECONDS = 5.0
|
|
INTERNAL_CANCEL_REASON = "text_test_case_complete"
|
|
AUTOMATIC_KNOWLEDGE_HINT = (
|
|
"你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
|
|
"回答资料事实时只根据检索内容，资料不足要明确说明。"
|
|
)
|
|
ON_DEMAND_KNOWLEDGE_HINT = (
|
|
"你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，"
|
|
"先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，"
|
|
"资料不足要明确说明。"
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class RawToolCall:
|
|
id: str
|
|
function_name: str
|
|
arguments: Any
|
|
result: Any = None
|
|
outcome: str = "error"
|
|
duration_ms: int = 0
|
|
_started_at: float = field(default_factory=time.monotonic, repr=False)
|
|
|
|
def as_result(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id,
|
|
"functionName": self.function_name,
|
|
"argumentsJson": json.dumps(
|
|
self.arguments, ensure_ascii=False, indent=2, default=str
|
|
),
|
|
"resultJson": json.dumps(
|
|
self.result, ensure_ascii=False, indent=2, default=str
|
|
),
|
|
"outcome": self.outcome,
|
|
"durationMs": self.duration_ms,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class RawTurnResult:
|
|
id: str
|
|
index: int
|
|
user_input: str
|
|
assistant_reply: str
|
|
tool_calls: list[RawToolCall]
|
|
image: TestImageRef | None = None
|
|
runtime_records: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class TextStartupResult:
|
|
"""Headless startup choices applied before the scripted user turns."""
|
|
|
|
greeting: str = ""
|
|
assistant_outputs: list[str] = field(default_factory=list)
|
|
auto_confirmed_messages: list[dict[str, Any]] = field(default_factory=list)
|
|
runtime_records: list[dict[str, Any]] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class TextCaseResult:
|
|
turns: list[RawTurnResult]
|
|
transcript: list[dict[str, str]]
|
|
startup: TextStartupResult = field(default_factory=TextStartupResult)
|
|
|
|
|
|
TurnCompleteCallback = Callable[[RawTurnResult], Awaitable[None]]
|
|
StartupCompleteCallback = Callable[[TextStartupResult], Awaitable[None]]
|
|
|
|
|
|
@dataclass
|
|
class _StartupBoundaryFrame(DataFrame):
|
|
"""Ordered marker queued behind every frame emitted by startup."""
|
|
|
|
reached: asyncio.Event
|
|
|
|
|
|
@dataclass
|
|
class _ResponseWindow:
|
|
active_llm: int = 0
|
|
active_assistant: int = 0
|
|
pending_tools: set[str] = field(default_factory=set)
|
|
activity_count: int = 0
|
|
last_activity_at: float = field(default_factory=time.monotonic)
|
|
|
|
def touch(self) -> None:
|
|
self.activity_count += 1
|
|
self.last_activity_at = time.monotonic()
|
|
|
|
@property
|
|
def idle(self) -> bool:
|
|
return self.active_llm == 0 and self.active_assistant == 0 and not self.pending_tools
|
|
|
|
|
|
def _opening_speech_from_message(message: Any) -> str:
|
|
"""Extract only the transcript emitted by Prompt's opening Message stage."""
|
|
|
|
if (
|
|
not isinstance(message, dict)
|
|
or message.get("type") != "transcript"
|
|
or message.get("source") != "prompt-opening-speech"
|
|
):
|
|
return ""
|
|
return str(message.get("content") or "").strip()
|
|
|
|
|
|
def _runtime_record_from_message(message: Any) -> dict[str, Any] | None:
|
|
"""Keep the same runtime-record envelope used by live conversation history."""
|
|
|
|
if (
|
|
not isinstance(message, dict)
|
|
or message.get("type") != "workflow-event"
|
|
or message.get("internalStage") is True
|
|
):
|
|
return None
|
|
return dict(message)
|
|
|
|
|
|
class _TextCaptureProcessor(FrameProcessor):
|
|
"""Capture assistant/tool output and expose a reliable response boundary."""
|
|
|
|
def __init__(self, context: LLMContext):
|
|
super().__init__()
|
|
self.context = context
|
|
self.window = _ResponseWindow()
|
|
self.outputs: list[str] = []
|
|
self.opening_speech_outputs: list[str] = []
|
|
self.runtime_records: list[dict[str, Any]] = []
|
|
self.tool_calls: list[RawToolCall] = []
|
|
self._tool_by_id: dict[str, RawToolCall] = {}
|
|
self.error: TestExecutionError | None = None
|
|
self.on_fixed_speech_complete: Callable[[], Awaitable[None]] | None = None
|
|
|
|
def fail(self, frame: ErrorFrame) -> None:
|
|
"""Release every wait even when Pipecat reports the error upstream."""
|
|
|
|
message = frame.error or "Pipeline 执行失败"
|
|
unmocked_tool = "UNMOCKED_TOOL_CALL" in message
|
|
self.error = TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR",
|
|
message=message,
|
|
stage=(
|
|
"tool"
|
|
if unmocked_tool
|
|
else "model"
|
|
if frame.processor
|
|
else "pipeline"
|
|
),
|
|
retryable=not unmocked_tool,
|
|
)
|
|
self.window.pending_tools.clear()
|
|
self.window.touch()
|
|
|
|
async def process_frame(self, frame, direction: FrameDirection):
|
|
await super().process_frame(frame, direction)
|
|
|
|
if isinstance(frame, TTSSpeakFrame):
|
|
text = frame.text.strip()
|
|
if text:
|
|
self.outputs.append(text)
|
|
if frame.append_to_context:
|
|
self.context.add_message({"role": "assistant", "content": text})
|
|
self.window.touch()
|
|
if self.on_fixed_speech_complete is not None:
|
|
await self.on_fixed_speech_complete()
|
|
return
|
|
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame):
|
|
opening_speech = _opening_speech_from_message(frame.message)
|
|
if opening_speech:
|
|
self.opening_speech_outputs.append(opening_speech)
|
|
runtime_record = _runtime_record_from_message(frame.message)
|
|
if runtime_record is not None:
|
|
self.runtime_records.append(runtime_record)
|
|
|
|
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
|
|
await frame.completion.mark_played()
|
|
self.window.touch()
|
|
return
|
|
|
|
if isinstance(frame, LLMFullResponseStartFrame):
|
|
self.window.active_llm += 1
|
|
self.window.touch()
|
|
elif isinstance(frame, LLMFullResponseEndFrame):
|
|
self.window.active_llm = max(0, self.window.active_llm - 1)
|
|
self.window.touch()
|
|
await self.push_frame(frame, direction)
|
|
# There is no TTS/output transport. This releases provider/tool
|
|
# follow-up logic that normally waits for a speaking boundary.
|
|
await self.push_frame(BotStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
|
|
return
|
|
elif isinstance(frame, FunctionCallInProgressFrame):
|
|
call = RawToolCall(
|
|
id=frame.tool_call_id,
|
|
function_name=frame.function_name,
|
|
arguments=frame.arguments or {},
|
|
)
|
|
self.tool_calls.append(call)
|
|
self._tool_by_id[call.id] = call
|
|
self.window.pending_tools.add(call.id)
|
|
self.window.touch()
|
|
elif isinstance(frame, FunctionCallResultFrame):
|
|
call = self._tool_by_id.get(frame.tool_call_id)
|
|
if call is None:
|
|
call = RawToolCall(
|
|
id=frame.tool_call_id,
|
|
function_name=frame.function_name,
|
|
arguments=frame.arguments or {},
|
|
)
|
|
self.tool_calls.append(call)
|
|
self._tool_by_id[call.id] = call
|
|
call.result = frame.result
|
|
call.duration_ms = max(0, round((time.monotonic() - call._started_at) * 1000))
|
|
status = frame.result.get("status") if isinstance(frame.result, dict) else None
|
|
call.outcome = "error" if status in {"error", "failed", "timeout"} else "success"
|
|
self.window.pending_tools.discard(call.id)
|
|
self.window.touch()
|
|
elif isinstance(frame, ErrorFrame):
|
|
self.fail(frame)
|
|
elif isinstance(frame, _StartupBoundaryFrame):
|
|
frame.reached.set()
|
|
self.window.touch()
|
|
elif isinstance(frame, (EndFrame, CancelFrame)):
|
|
self.window.touch()
|
|
|
|
await self.push_frame(frame, direction)
|
|
|
|
|
|
def _resolve_startup_greeting(
|
|
shared_greeting: str,
|
|
opening_speech_outputs: list[str],
|
|
) -> str:
|
|
"""Return the opening speech shown by the headless run.
|
|
|
|
Prompt confirmation mode deliberately suppresses the shared greeting
|
|
because its atomic Message stage owns both speech and the dialog. In text
|
|
tests the fixed speech transcript carries a dedicated source marker, so
|
|
preserve that exact output instead of guessing from unrelated fixed speech.
|
|
"""
|
|
|
|
greeting = shared_greeting.strip()
|
|
if greeting:
|
|
return greeting
|
|
return next(
|
|
(text.strip() for text in opening_speech_outputs if text.strip()),
|
|
"",
|
|
)
|
|
|
|
|
|
def _context_messages(context_turns: list[ContextTurn]) -> list[dict[str, Any]]:
|
|
messages: list[dict[str, Any]] = []
|
|
latest_call_by_name: dict[str, str] = {}
|
|
for index, turn in enumerate(context_turns):
|
|
if turn.role in {"agent", "user"}:
|
|
messages.append(
|
|
{
|
|
"role": "assistant" if turn.role == "agent" else "user",
|
|
"content": turn.content,
|
|
}
|
|
)
|
|
continue
|
|
|
|
call_id = turn.tool_call_id or latest_call_by_name.get(turn.tool_name or "")
|
|
call_id = call_id or f"context_tool_{index}_{uuid4().hex[:8]}"
|
|
if turn.role == "tool_call":
|
|
latest_call_by_name[turn.tool_name or ""] = call_id
|
|
messages.append(
|
|
{
|
|
"role": "assistant",
|
|
"tool_calls": [
|
|
{
|
|
"id": call_id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": turn.tool_name,
|
|
"arguments": json.dumps(
|
|
json.loads(turn.content), ensure_ascii=False
|
|
),
|
|
},
|
|
}
|
|
],
|
|
}
|
|
)
|
|
else:
|
|
result_payload = json.loads(turn.content)
|
|
if turn.is_error:
|
|
result_payload = (
|
|
{**result_payload, "status": "error"}
|
|
if isinstance(result_payload, dict)
|
|
else {"status": "error", "data": result_payload}
|
|
)
|
|
messages.append(
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": call_id,
|
|
"content": json.dumps(result_payload, ensure_ascii=False),
|
|
}
|
|
)
|
|
return messages
|
|
|
|
|
|
def _transcript_content(message: dict[str, Any]) -> str:
|
|
"""Keep evaluator transcripts readable without persisting image data URIs."""
|
|
|
|
content = message.get("content")
|
|
if isinstance(content, str):
|
|
return content.strip()
|
|
if not isinstance(content, list):
|
|
return ""
|
|
text_parts = [
|
|
str(part.get("text") or "").strip()
|
|
for part in content
|
|
if isinstance(part, dict)
|
|
and part.get("type") in {"text", "input_text"}
|
|
and str(part.get("text") or "").strip()
|
|
]
|
|
has_image = any(
|
|
isinstance(part, dict)
|
|
and part.get("type") in {"image_url", "input_image"}
|
|
for part in content
|
|
)
|
|
if has_image:
|
|
text_parts.append("[用户提交了一张测试图片]")
|
|
return "\n".join(text_parts).strip()
|
|
|
|
|
|
async def _wait_for_quiescence(
|
|
capture: _TextCaptureProcessor,
|
|
runner_task: asyncio.Task[None],
|
|
activity_marker: int,
|
|
*,
|
|
require_activity: bool = True,
|
|
) -> None:
|
|
while True:
|
|
if capture.error is not None:
|
|
raise capture.error
|
|
if runner_task.done():
|
|
await runner_task
|
|
return
|
|
has_activity = capture.window.activity_count > activity_marker
|
|
settled = time.monotonic() - capture.window.last_activity_at >= IDLE_SETTLE_SECONDS
|
|
if capture.window.idle and settled and (has_activity or not require_activity):
|
|
return
|
|
await asyncio.sleep(0.05)
|
|
|
|
|
|
async def _wait_for_startup(
|
|
worker: PipelineWorker,
|
|
capture: _TextCaptureProcessor,
|
|
runner_task: asyncio.Task[None],
|
|
activity_marker: int,
|
|
) -> None:
|
|
"""Wait until startup-queued frames and any entry LLM reply are complete."""
|
|
|
|
boundary = asyncio.Event()
|
|
await worker.queue_frame(_StartupBoundaryFrame(reached=boundary))
|
|
boundary_waiter = asyncio.create_task(boundary.wait())
|
|
done, _pending = await asyncio.wait(
|
|
{boundary_waiter, runner_task},
|
|
timeout=WORKER_START_TIMEOUT_SECONDS,
|
|
return_when=asyncio.FIRST_COMPLETED,
|
|
)
|
|
if not done:
|
|
boundary_waiter.cancel()
|
|
raise TestExecutionError(
|
|
code="STARTUP_DRAIN_TIMEOUT",
|
|
message="开场阶段未能进入稳定状态",
|
|
stage="pipeline",
|
|
retryable=True,
|
|
)
|
|
if boundary_waiter not in done:
|
|
boundary_waiter.cancel()
|
|
await asyncio.gather(boundary_waiter, return_exceptions=True)
|
|
await _wait_for_quiescence(
|
|
capture,
|
|
runner_task,
|
|
activity_marker,
|
|
require_activity=False,
|
|
)
|
|
|
|
|
|
class TextPipelineRunner:
|
|
"""Execute all fixed input turns in one isolated, transportless pipeline."""
|
|
|
|
async def run(
|
|
self,
|
|
cfg: AssistantConfig,
|
|
definition: TestCaseDefinition,
|
|
*,
|
|
timeout_seconds: int,
|
|
image_assets: dict[str, bytes] | None = None,
|
|
on_turn_complete: TurnCompleteCallback | None = None,
|
|
on_startup_complete: StartupCompleteCallback | None = None,
|
|
) -> TextCaseResult:
|
|
try:
|
|
async with asyncio.timeout(timeout_seconds):
|
|
return await self._run(
|
|
cfg,
|
|
definition,
|
|
image_assets=image_assets or {},
|
|
on_turn_complete=on_turn_complete,
|
|
on_startup_complete=on_startup_complete,
|
|
)
|
|
except TimeoutError as exc:
|
|
raise TestExecutionError(
|
|
code="PIPELINE_TIMEOUT",
|
|
message=f"测试用例执行超过 {timeout_seconds} 秒",
|
|
stage="pipeline",
|
|
retryable=True,
|
|
) from exc
|
|
|
|
async def _run(
|
|
self,
|
|
cfg: AssistantConfig,
|
|
definition: TestCaseDefinition,
|
|
*,
|
|
image_assets: dict[str, bytes],
|
|
on_turn_complete: TurnCompleteCallback | None,
|
|
on_startup_complete: StartupCompleteCallback | None,
|
|
) -> TextCaseResult:
|
|
if cfg.runtimeMode != "pipeline":
|
|
raise TestExecutionError(
|
|
code="UNSUPPORTED_RUNTIME_MODE",
|
|
message="第一版文本测试只支持 Pipeline 运行模式",
|
|
stage="pipeline",
|
|
)
|
|
if cfg.type not in {"prompt", "workflow"}:
|
|
raise TestExecutionError(
|
|
code="UNSUPPORTED_ASSISTANT_TYPE",
|
|
message=f"第一版文本测试暂不支持 {cfg.type} 类型助手",
|
|
stage="pipeline",
|
|
)
|
|
|
|
brain = build_brain(cfg)
|
|
seed_messages = _context_messages(definition.context_turns)
|
|
knowledge_config = cfg.knowledge_retrieval_config or {}
|
|
knowledge_mode = str(knowledge_config.get("mode") or "automatic")
|
|
|
|
def with_knowledge_hint(text: str) -> str:
|
|
if cfg.type != "prompt" or not cfg.knowledge_base_id:
|
|
return text
|
|
hint = (
|
|
AUTOMATIC_KNOWLEDGE_HINT
|
|
if knowledge_mode == "automatic"
|
|
else ON_DEMAND_KNOWLEDGE_HINT
|
|
)
|
|
return "\n\n".join(part for part in (text, hint) if part)
|
|
|
|
system_prompt = with_knowledge_hint(brain.system_prompt(cfg))
|
|
context = LLMContext(
|
|
messages=(
|
|
seed_messages
|
|
if cfg.type == "workflow"
|
|
else [{"role": "system", "content": system_prompt}, *seed_messages]
|
|
)
|
|
)
|
|
|
|
graph_settings = cfg.graph.get("settings") or {}
|
|
default_resource = cfg.workflow_model_resources.get(
|
|
str(graph_settings.get("defaultLlmResourceId") or "")
|
|
)
|
|
llm_cfg = (
|
|
config_with_resource(cfg, default_resource)
|
|
if cfg.type == "workflow" and default_resource is not None
|
|
else cfg
|
|
)
|
|
llm = brain.build_llm(llm_cfg, context)
|
|
llm_services: dict[str, FrameProcessor] = {}
|
|
current_llm_service: FrameProcessor = llm
|
|
if cfg.type == "workflow":
|
|
llm, llm_services, current_llm_service = build_workflow_llm_switcher(cfg, llm)
|
|
|
|
aggregators = LLMContextAggregatorPair(context)
|
|
user_aggregator = aggregators.user()
|
|
assistant_aggregator = aggregators.assistant()
|
|
capture = _TextCaptureProcessor(context)
|
|
knowledge = KnowledgeRetrievalProcessor(
|
|
cfg.knowledge_base_id if knowledge_mode == "automatic" else None,
|
|
top_n=int(knowledge_config.get("top_n", knowledge_config.get("topN", 5))),
|
|
score_threshold=float(
|
|
knowledge_config.get(
|
|
"score_threshold", knowledge_config.get("scoreThreshold", 0.0)
|
|
)
|
|
),
|
|
)
|
|
pipeline = Pipeline(
|
|
[
|
|
user_aggregator,
|
|
UserTurnRoutingProcessor(brain),
|
|
knowledge,
|
|
llm,
|
|
capture,
|
|
assistant_aggregator,
|
|
]
|
|
)
|
|
worker = PipelineWorker(
|
|
pipeline,
|
|
params=PipelineParams(enable_metrics=False),
|
|
enable_rtvi=False,
|
|
enable_turn_tracking=False,
|
|
idle_timeout_secs=None,
|
|
)
|
|
runner = WorkerRunner(handle_sigint=False, check_dangling_tasks=False)
|
|
worker_started = asyncio.Event()
|
|
|
|
@worker.event_handler("on_pipeline_started")
|
|
async def on_pipeline_started(_worker, _frame):
|
|
worker_started.set()
|
|
|
|
@worker.event_handler("on_pipeline_error")
|
|
async def on_pipeline_error(_worker, frame):
|
|
capture.fail(frame)
|
|
|
|
executor_holder: dict[str, MockToolExecutor] = {}
|
|
text_client_tools = TextClientToolPort()
|
|
workflow_vision_scope: dict[str, Any] = {"enabled": False}
|
|
|
|
def create_test_executor(store) -> MockToolExecutor:
|
|
executor = MockToolExecutor(store)
|
|
executor_holder["executor"] = executor
|
|
return executor
|
|
|
|
async def queue_call_end(reason: str) -> None:
|
|
logger.debug(f"文本测试收到会话结束请求: {reason}")
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message=call_end_trace_event(
|
|
"ended",
|
|
reason=reason,
|
|
)
|
|
)
|
|
)
|
|
await worker.queue_frame(EndFrame(reason=reason))
|
|
|
|
async def record_call_end_state(
|
|
state: str,
|
|
interruptible: bool,
|
|
) -> None:
|
|
await worker.queue_frame(
|
|
OutputTransportMessageUrgentFrame(
|
|
message=call_end_trace_event(
|
|
state,
|
|
reason=call_end.reason,
|
|
interruptible=interruptible,
|
|
)
|
|
)
|
|
)
|
|
|
|
call_end = CallEndCoordinator(
|
|
queue_call_end,
|
|
on_state_change=record_call_end_state,
|
|
)
|
|
|
|
async def finish_fixed_speech_if_ending() -> None:
|
|
if call_end.ending:
|
|
await call_end.finish()
|
|
|
|
capture.on_fixed_speech_complete = finish_fixed_speech_if_ending
|
|
current_service = current_llm_service
|
|
|
|
async def switch_services(
|
|
llm_resource_id: str | None,
|
|
_asr_resource_id: str | None,
|
|
_tts_resource_id: str | None,
|
|
) -> None:
|
|
nonlocal current_service
|
|
target = (
|
|
llm_services.get(llm_resource_id)
|
|
if llm_resource_id
|
|
else current_llm_service
|
|
)
|
|
if target is None:
|
|
raise ValueError(f"Workflow LLM 资源未加载:{llm_resource_id}")
|
|
if target is current_service:
|
|
return
|
|
await worker.queue_frame(ManuallySwitchServiceFrame(service=target))
|
|
current_service = target
|
|
|
|
def set_system_prompt(text: str) -> None:
|
|
messages = context.get_messages()
|
|
text = with_knowledge_hint(text)
|
|
if messages and messages[0].get("role") == "system":
|
|
messages[0] = {"role": "system", "content": text}
|
|
else:
|
|
messages.insert(0, {"role": "system", "content": text})
|
|
|
|
knowledge_schema: FunctionSchema | None = None
|
|
if (
|
|
cfg.type == "prompt"
|
|
and cfg.knowledge_base_id
|
|
and knowledge_mode == "on_demand"
|
|
):
|
|
knowledge_schema = FunctionSchema(
|
|
name="search_knowledge_base",
|
|
description=(
|
|
"在当前助手绑定的知识库中检索资料。"
|
|
f"知识库：{cfg.knowledge_base_name}。"
|
|
f"{cfg.knowledge_base_description}"
|
|
),
|
|
properties={
|
|
"query": {
|
|
"type": "string",
|
|
"description": "完整问题或检索关键词",
|
|
}
|
|
},
|
|
required=["query"],
|
|
)
|
|
|
|
async def search_bound_knowledge(params: FunctionCallParams) -> None:
|
|
query = str(params.arguments.get("query") or "").strip()
|
|
if not query:
|
|
await params.result_callback(
|
|
{"status": "error", "message": "检索问题为空"}
|
|
)
|
|
return
|
|
try:
|
|
async with SessionLocal() as session:
|
|
results = await search_knowledge(
|
|
session,
|
|
cfg.knowledge_base_id or "",
|
|
query,
|
|
top_k=int(
|
|
knowledge_config.get(
|
|
"top_n",
|
|
knowledge_config.get("topN", 5),
|
|
)
|
|
),
|
|
score_threshold=float(
|
|
knowledge_config.get(
|
|
"score_threshold",
|
|
knowledge_config.get("scoreThreshold", 0.0),
|
|
)
|
|
),
|
|
)
|
|
await params.result_callback(
|
|
{"status": "ok", "results": results}
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - return tool errors to LLM
|
|
logger.warning(f"文本测试知识库检索失败: {exc}")
|
|
await params.result_callback(
|
|
{"status": "error", "message": "知识库检索暂时不可用"}
|
|
)
|
|
|
|
llm.register_function("search_knowledge_base", search_bound_knowledge)
|
|
|
|
def set_tools(schemas=None) -> None:
|
|
visible = list(schemas or [])
|
|
if knowledge_schema is not None:
|
|
visible.append(knowledge_schema)
|
|
if visible:
|
|
context.set_tools(ToolsSchema(standard_tools=visible))
|
|
else:
|
|
context.set_tools()
|
|
|
|
await brain.setup(
|
|
cfg,
|
|
BrainRuntime(
|
|
context=context,
|
|
llm=llm,
|
|
queue_frame=worker.queue_frame,
|
|
set_system_prompt=set_system_prompt,
|
|
set_tools=set_tools,
|
|
call_end=call_end,
|
|
session_id=f"test_{uuid4().hex}",
|
|
client_tools=text_client_tools,
|
|
worker=worker,
|
|
context_aggregator=aggregators,
|
|
transport=None,
|
|
switch_services=switch_services,
|
|
set_knowledge_scope=knowledge.set_scope,
|
|
set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
|
|
vision_function=None,
|
|
set_input_enabled=lambda _enabled: None,
|
|
apply_turn_config=lambda _enabled, _config: asyncio.sleep(0),
|
|
flow_global_functions=[],
|
|
tool_executor_factory=create_test_executor,
|
|
),
|
|
)
|
|
executor = executor_holder["executor"]
|
|
|
|
# A transportless text run cannot synthesize audio, but the configured
|
|
# greeting has already completed before scripted inputs begin. Mirror
|
|
# production's provider-safe context fact so the model does not repeat it.
|
|
greeting = str(await brain.greeting(cfg) or "").strip()
|
|
if greeting and brain.spec.owns_context:
|
|
brain.prepare_greeting_context(greeting, context)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_turn_started")
|
|
async def on_assistant_turn_started(_aggregator):
|
|
capture.window.active_assistant += 1
|
|
capture.window.touch()
|
|
await brain.on_assistant_text_start(uuid4().hex)
|
|
|
|
@assistant_aggregator.event_handler("on_assistant_turn_stopped")
|
|
async def on_assistant_turn_stopped(_aggregator, message):
|
|
content = str(message.content or "").strip()
|
|
capture.window.active_assistant = max(
|
|
0, capture.window.active_assistant - 1
|
|
)
|
|
if content:
|
|
capture.outputs.append(content)
|
|
capture.window.touch()
|
|
await brain.on_assistant_text_end(
|
|
uuid4().hex,
|
|
content,
|
|
bool(getattr(message, "interrupted", False)),
|
|
)
|
|
if call_end.ending:
|
|
# Text tests have no audio transport. A fully aggregated
|
|
# response is therefore the equivalent of playback ending.
|
|
await call_end.finish()
|
|
|
|
all_tool_behaviors = [
|
|
behavior
|
|
for turn in definition.turns
|
|
for behavior in turn.behaviors
|
|
if behavior.type == "tool_call"
|
|
]
|
|
executor.set_turn_behaviors(all_tool_behaviors)
|
|
|
|
runner_task: asyncio.Task[None] | None = None
|
|
try:
|
|
startup_runtime_record_marker = len(capture.runtime_records)
|
|
await brain.run_preflight()
|
|
if executor.unmocked_calls:
|
|
raise TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL",
|
|
message="启动阶段调用了未配置 Mock 的工具: "
|
|
+ "、".join(executor.unmocked_calls),
|
|
stage="tool",
|
|
)
|
|
|
|
await runner.add_workers(worker)
|
|
runner_task = asyncio.create_task(
|
|
runner.run(), name=f"text-test-{uuid4().hex[:12]}"
|
|
)
|
|
await asyncio.wait_for(
|
|
worker_started.wait(), timeout=WORKER_START_TIMEOUT_SECONDS
|
|
)
|
|
|
|
marker = capture.window.activity_count
|
|
await brain.on_connected(greeting_pending=False)
|
|
await brain.on_client_ready()
|
|
await _wait_for_startup(
|
|
worker,
|
|
capture,
|
|
runner_task,
|
|
marker,
|
|
)
|
|
startup_greeting = _resolve_startup_greeting(
|
|
greeting,
|
|
capture.opening_speech_outputs,
|
|
)
|
|
startup = TextStartupResult(
|
|
greeting=startup_greeting,
|
|
assistant_outputs=list(capture.outputs),
|
|
auto_confirmed_messages=list(
|
|
text_client_tools.auto_confirmed_messages
|
|
),
|
|
runtime_records=list(
|
|
capture.runtime_records[startup_runtime_record_marker:]
|
|
),
|
|
)
|
|
if on_startup_complete is not None:
|
|
await on_startup_complete(startup)
|
|
# Startup messages initialize the session but are not a scripted
|
|
# user turn result in the fixed-input editor contract.
|
|
capture.outputs.clear()
|
|
capture.tool_calls.clear()
|
|
|
|
results: list[RawTurnResult] = []
|
|
for index, turn in enumerate(definition.turns):
|
|
if worker.has_finished():
|
|
raise TestExecutionError(
|
|
code="PIPELINE_ENDED_EARLY",
|
|
message=f"Pipeline 在第 {index + 1} 轮输入前已经结束",
|
|
stage="pipeline",
|
|
retryable=False,
|
|
)
|
|
result = await self._run_turn(
|
|
cfg,
|
|
worker,
|
|
runner_task,
|
|
capture,
|
|
executor,
|
|
turn,
|
|
index,
|
|
image_assets=image_assets,
|
|
workflow_vision_scope=workflow_vision_scope,
|
|
)
|
|
results.append(result)
|
|
if on_turn_complete is not None:
|
|
await on_turn_complete(result)
|
|
|
|
transcript = []
|
|
for message in context.get_messages():
|
|
if message.get("role") not in {"user", "assistant"}:
|
|
continue
|
|
content = _transcript_content(message)
|
|
if content:
|
|
transcript.append(
|
|
{
|
|
"role": str(message.get("role") or ""),
|
|
"content": content,
|
|
}
|
|
)
|
|
return TextCaseResult(
|
|
turns=results,
|
|
transcript=transcript,
|
|
startup=startup,
|
|
)
|
|
except TestExecutionError:
|
|
raise
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
raise TestExecutionError(
|
|
code="PIPELINE_EXECUTION_FAILED",
|
|
message=str(exc) or type(exc).__name__,
|
|
stage="model",
|
|
retryable=True,
|
|
) from exc
|
|
finally:
|
|
if not worker.has_finished():
|
|
await worker.cancel(reason=INTERNAL_CANCEL_REASON)
|
|
if runner_task is not None:
|
|
await asyncio.gather(runner_task, return_exceptions=True)
|
|
|
|
async def _run_turn(
|
|
self,
|
|
cfg: AssistantConfig,
|
|
worker: PipelineWorker,
|
|
runner_task: asyncio.Task[None],
|
|
capture: _TextCaptureProcessor,
|
|
executor: MockToolExecutor,
|
|
turn: FixedInputTurn,
|
|
index: int,
|
|
*,
|
|
image_assets: dict[str, bytes],
|
|
workflow_vision_scope: dict[str, Any],
|
|
) -> RawTurnResult:
|
|
tool_behaviors = [
|
|
behavior for behavior in turn.behaviors if behavior.type == "tool_call"
|
|
]
|
|
executor.set_turn_behaviors(tool_behaviors)
|
|
activity_marker = capture.window.activity_count
|
|
output_marker = len(capture.outputs)
|
|
tool_marker = len(capture.tool_calls)
|
|
runtime_record_marker = len(capture.runtime_records)
|
|
user_message: dict[str, Any] = {
|
|
"role": "user",
|
|
"content": turn.user_input,
|
|
}
|
|
if turn.image is not None:
|
|
image_bytes = image_assets.get(turn.image.asset_id)
|
|
if image_bytes is None:
|
|
raise TestExecutionError(
|
|
code="TEST_IMAGE_MISSING",
|
|
message=f"测试图片不存在: {turn.image.asset_id}",
|
|
stage="pipeline",
|
|
retryable=False,
|
|
)
|
|
try:
|
|
user_message = await build_image_user_message(
|
|
cfg,
|
|
image_bytes,
|
|
turn.user_input,
|
|
workflow_scope=(
|
|
workflow_vision_scope if cfg.type == "workflow" else None
|
|
),
|
|
)
|
|
except ValueError as exc:
|
|
raise TestExecutionError(
|
|
code="TEST_IMAGE_UNSUPPORTED",
|
|
message=str(exc),
|
|
stage="model",
|
|
retryable=False,
|
|
) from exc
|
|
await worker.queue_frame(
|
|
LLMMessagesAppendFrame(messages=[user_message], run_llm=True)
|
|
)
|
|
await _wait_for_quiescence(capture, runner_task, activity_marker)
|
|
if executor.unmocked_calls:
|
|
raise TestExecutionError(
|
|
code="UNMOCKED_TOOL_CALL",
|
|
message="本轮调用了未配置 Mock 的工具: "
|
|
+ "、".join(executor.unmocked_calls),
|
|
stage="tool",
|
|
)
|
|
return RawTurnResult(
|
|
id=turn.id,
|
|
index=index,
|
|
user_input=turn.user_input,
|
|
image=turn.image,
|
|
assistant_reply="\n\n".join(capture.outputs[output_marker:]).strip(),
|
|
tool_calls=list(capture.tool_calls[tool_marker:]),
|
|
runtime_records=list(
|
|
capture.runtime_records[runtime_record_marker:]
|
|
),
|
|
)
|