wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(testing): add fixed tool simulations

Browse Source
This commit is contained in:
Xin Wang
2026-08-26 09:36:37 +08:00
parent 26582bfb87
commit 239b9027ca
16 changed files with 1525 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/config_resolver.py
+41 -4
View File
@@ -22,7 +22,7 @@ from models import (
RuntimeToolBinding,
)
from services.node_specs import copy_graph, graph_references
from sqlalchemy import select
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -135,24 +135,56 @@ def _runtime_tool_ids(
)
def _implicit_startup_tool_names(startup: dict | None) -> list[str]:
"""Return platform client tools used by built-in startup experiences.
A confirmation opening is configured directly on the assistant instead of
through an AssistantToolBinding, but text tests still need to simulate the
underlying client interaction explicitly.
"""
value = startup or {}
opening_mode = value.get("opening_mode") or value.get("openingMode")
opening_message = value.get("opening_message") or value.get("openingMessage")
if opening_mode == "confirmation" and isinstance(opening_message, dict):
return ["show_message"]
return []
async def _tools_for(
session: AsyncSession,
assistant: Assistant,
tool_ids: list[str],
implicit_function_names: list[str] | None = None,
) -> list[RuntimeTool]:
if assistant.type not in {"prompt", "workflow"} or not tool_ids:
function_names = list(dict.fromkeys(implicit_function_names or []))
if assistant.type not in {"prompt", "workflow"} or not (
tool_ids or function_names
):
return []
identity_filters = []
if tool_ids:
identity_filters.append(Tool.id.in_(tool_ids))
if function_names:
identity_filters.append(Tool.function_name.in_(function_names))
rows = (
await session.execute(
select(Tool)
.where(
Tool.id.in_(tool_ids),
or_(*identity_filters),
Tool.status == "active",
)
)
).scalars().all()
tool_by_id = {tool.id: tool for tool in rows}
tool_by_function = {tool.function_name: tool for tool in rows}
tools = [tool_by_id[tool_id] for tool_id in tool_ids if tool_id in tool_by_id]
seen_tool_ids = {tool.id for tool in tools}
for function_name in function_names:
tool = tool_by_function.get(function_name)
if tool is not None and tool.id not in seen_tool_ids:
tools.append(tool)
seen_tool_ids.add(tool.id)
server_ids = {
str(tool.mcp_server_id)
for tool in tools
@@ -268,7 +300,12 @@ async def resolve_runtime_config(
prompt_tool_bindings = await _bound_tools(session, assistant.id)
runtime_tool_ids = _runtime_tool_ids(prompt_tool_bindings, assistant.startup or {})
runtime_tools = await _tools_for(session, assistant, runtime_tool_ids)
runtime_tools = await _tools_for(
session,
assistant,
runtime_tool_ids,
_implicit_startup_tool_names(assistant.startup or {}),
)
return AssistantConfig(
name=assistant.name,
backend/services/test_runs/mock_tools.py
+168 -46
View File
@@ -1,4 +1,4 @@
"""Test-only tool executor that never reaches external side effects."""
"""Strict, deterministic tool simulations for text test runs."""
from __future__ import annotations
@@ -9,25 +9,141 @@ from typing import Any
from models import RuntimeTool, RuntimeToolBinding
from services.runtime_variables import DynamicVariableStore
from services.tool_executor import ToolExecutionError, ToolExecutor
from test_schemas import ToolCallExpectedBehavior
from test_schemas import ToolSimulation, ToolSimulationResponse, ToolSimulationStep
TOOL_SIMULATION_ERROR_CODES = {
"UNMOCKED_TOOL_CALL",
"TOOL_SIMULATION_EXHAUSTED",
"TOOL_SIMULATION_SCOPE_MISMATCH",
}
class ToolSimulationError(ToolExecutionError):
"""A non-retryable test configuration error raised at tool-call time."""
def __init__(self, code: str, message: str):
self.code = code
super().__init__(f"{code}: {message}")
class ToolSimulationResolver:
"""Resolve one case's fixed or ordered simulated tool responses."""
def __init__(self, simulations: list[ToolSimulation]):
self._by_tool_id = {item.tool_id: item for item in simulations}
self._by_function = {item.function_name: item for item in simulations}
self._positions = {item.tool_id: 0 for item in simulations}
self._phase = "startup"
self._turn_id: str | None = None
self._failure: ToolSimulationError | None = None
def enter_startup(self) -> None:
self._phase = "startup"
self._turn_id = None
def enter_turn(self, turn_id: str) -> None:
self._phase = "turn"
self._turn_id = turn_id
async def resolve(
self,
*,
tool_id: str | None,
function_name: str,
) -> dict[str, Any]:
simulation = (
self._by_tool_id.get(tool_id or "")
or self._by_function.get(function_name)
)
if simulation is None:
raise self._record_failure(
"UNMOCKED_TOOL_CALL",
f"工具 {function_name} 未配置工具模拟",
)
response: ToolSimulationResponse | ToolSimulationStep
if simulation.mode == "fixed":
response = simulation.response
else:
position = self._positions[simulation.tool_id]
if position >= len(simulation.steps):
raise self._record_failure(
"TOOL_SIMULATION_EXHAUSTED",
f"工具 {function_name} 的模拟响应已耗尽（第 {position + 1} 次调用）",
)
step = simulation.steps[position]
if not self._scope_matches(step):
raise self._record_failure(
"TOOL_SIMULATION_SCOPE_MISMATCH",
(
f"工具 {function_name} 的第 {position + 1} 条模拟步骤"
f"期望发生在{self._scope_label(step)}，"
f"实际发生在{self._current_scope_label()}"
),
)
response = step
self._positions[simulation.tool_id] = position + 1
if response.delay_ms:
await asyncio.sleep(response.delay_ms / 1000)
return self._normalized_result(response)
@property
def failure(self) -> ToolSimulationError | None:
return self._failure
def raise_if_failed(self) -> None:
"""Re-raise errors swallowed by a higher-level client message stage."""
if self._failure is not None:
raise self._failure
def _record_failure(self, code: str, message: str) -> ToolSimulationError:
error = ToolSimulationError(code, message)
if self._failure is None:
self._failure = error
return error
def _scope_matches(self, step: ToolSimulationStep) -> bool:
if step.scope.phase == "any":
return True
if step.scope.phase == "startup":
return self._phase == "startup"
return self._phase == "turn" and step.scope.turn_id == self._turn_id
@staticmethod
def _scope_label(step: ToolSimulationStep) -> str:
if step.scope.phase == "any":
return "任意阶段"
if step.scope.phase == "startup":
return "启动阶段"
return f"轮次 {step.scope.turn_id}"
def _current_scope_label(self) -> str:
if self._phase == "startup":
return "启动阶段"
return f"轮次 {self._turn_id or 'unknown'}"
@staticmethod
def _normalized_result(response: ToolSimulationResponse) -> dict[str, Any]:
payload = json.loads(response.body)
status = "ok" if response.outcome == "success" else "error"
if isinstance(payload, dict):
return {**payload, "status": status}
return {"status": status, "data": payload}
class MockToolExecutor(ToolExecutor):
"""Resolve the active turn's configured mocks and reject everything else."""
"""Execute external runtime tools exclusively through one test resolver."""
def __init__(self, store: DynamicVariableStore):
super().__init__(store)
self._mocks_by_tool_id: dict[str, ToolCallExpectedBehavior] = {}
self._mocks_by_function: dict[str, ToolCallExpectedBehavior] = {}
self.unmocked_calls: list[str] = []
def set_turn_behaviors(
def __init__(
self,
behaviors: list[ToolCallExpectedBehavior],
) -> None:
self._mocks_by_tool_id = {item.tool_id: item for item in behaviors}
self._mocks_by_function = {item.function_name: item for item in behaviors}
self.unmocked_calls = []
store: DynamicVariableStore,
resolver: ToolSimulationResolver,
):
super().__init__(store)
self._resolver = resolver
async def execute(
self,
@@ -37,26 +153,10 @@ class MockToolExecutor(ToolExecutor):
binding: RuntimeToolBinding | None = None,
result_assignments: dict[str, str] | None = None,
) -> dict[str, Any]:
behavior = self._mocks_by_tool_id.get(tool.id) or self._mocks_by_function.get(
tool.function_name
result = await self._resolver.resolve(
tool_id=tool.id,
function_name=tool.function_name,
)
if behavior is None:
self.unmocked_calls.append(tool.function_name)
raise ToolExecutionError(
f"UNMOCKED_TOOL_CALL: 工具 {tool.function_name} 未配置 Mock 返回值"
)
delay_seconds = behavior.mock_response.delay_ms / 1000
if delay_seconds:
await asyncio.sleep(delay_seconds)
payload = json.loads(behavior.mock_response.body)
status = "ok" if behavior.mock_response.outcome == "success" else "error"
if isinstance(payload, dict):
result: dict[str, Any] = dict(payload)
result["status"] = status
else:
result = {"status": status, "data": payload}
if result.get("status") != "ok":
return {**result, "updated_variables": []}
return self._apply_result_assignments(
@@ -71,13 +171,16 @@ class MockToolExecutor(ToolExecutor):
class TextClientToolPort:
"""Automatically acknowledge built-in message stages in text tests."""
"""Return configured client-tool results without a browser transport."""
def __init__(self) -> None:
def __init__(self, resolver: ToolSimulationResolver) -> None:
self._resolver = resolver
self.auto_confirmed_messages: list[dict[str, Any]] = []
def supports(self, function_name: str) -> bool:
return function_name == "show_message"
def supports(self, _function_name: str) -> bool:
# Test runs advertise the virtual client, then fail strictly in call().
# Returning False would silently skip an unconfigured client tool.
return True
async def call(
self,
@@ -85,12 +188,31 @@ class TextClientToolPort:
arguments: dict[str, Any],
**_kwargs: Any,
) -> dict[str, Any]:
if function_name == "show_message":
self.auto_confirmed_messages.append(dict(arguments))
return {
"status": "ok",
"data": {"action": "confirmed", "arguments": arguments},
}
raise ToolExecutionError(
f"UNMOCKED_CLIENT_TOOL: 客户端工具 {function_name} 不能脱离 Mock 执行"
resolved = await self._resolver.resolve(
tool_id=None,
function_name=function_name,
)
status = str(resolved.get("status") or "error")
payload = (
resolved.get("data")
if set(resolved).issubset({"status", "data"}) and "data" in resolved
else {key: value for key, value in resolved.items() if key != "status"}
)
# ClientToolPort mirrors the browser transport envelope. The JSON
# configured in the editor remains the client handler's root object.
result: dict[str, Any] = {"status": status, "data": payload}
if (
status != "ok"
and isinstance(payload, dict)
and payload.get("message") is not None
):
result["message"] = str(payload["message"])
data = result.get("data")
action = data.get("action") if isinstance(data, dict) else None
if (
function_name == "show_message"
and result.get("status") == "ok"
and action == "confirmed"
):
self.auto_confirmed_messages.append(dict(arguments))
return result
backend/services/test_runs/text_runner.py
+39 -41
View File
@@ -21,7 +21,6 @@ from pipecat.frames.frames import (
ErrorFrame,
FunctionCallInProgressFrame,
FunctionCallResultFrame,
LLMContextFrame,
LLMFullResponseEndFrame,
LLMFullResponseStartFrame,
LLMMessagesAppendFrame,
@@ -54,7 +53,13 @@ from services.pipecat.processors import (
from services.pipecat.service_factory import config_with_resource
from services.pipecat.workflow_services import build_workflow_llm_switcher
from services.test_runs.errors import TestExecutionError
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from services.test_runs.mock_tools import (
TOOL_SIMULATION_ERROR_CODES,
MockToolExecutor,
TextClientToolPort,
ToolSimulationError,
ToolSimulationResolver,
)
from services.vision import build_image_user_message
from test_schemas import ContextTurn, FixedInputTurn, TestCaseDefinition, TestImageRef
@@ -197,18 +202,21 @@ class _TextCaptureProcessor(FrameProcessor):
"""Release every wait even when Pipecat reports the error upstream."""
message = frame.error or "Pipeline 执行失败"
unmocked_tool = "UNMOCKED_TOOL_CALL" in message
simulation_code = next(
(code for code in TOOL_SIMULATION_ERROR_CODES if code in message),
None,
)
self.error = TestExecutionError(
code="UNMOCKED_TOOL_CALL" if unmocked_tool else "PIPELINE_ERROR",
code=simulation_code or "PIPELINE_ERROR",
message=message,
stage=(
"tool"
if unmocked_tool
if simulation_code
else "model"
if frame.processor
else "pipeline"
),
retryable=not unmocked_tool,
retryable=simulation_code is None,
)
self.window.pending_tools.clear()
self.window.touch()
@@ -573,14 +581,12 @@ class TextPipelineRunner:
async def on_pipeline_error(_worker, frame):
capture.fail(frame)
executor_holder: dict[str, MockToolExecutor] = {}
text_client_tools = TextClientToolPort()
simulation_resolver = ToolSimulationResolver(definition.tool_simulations)
text_client_tools = TextClientToolPort(simulation_resolver)
workflow_vision_scope: dict[str, Any] = {"enabled": False}
def create_test_executor(store) -> MockToolExecutor:
executor = MockToolExecutor(store)
executor_holder["executor"] = executor
return executor
return MockToolExecutor(store, simulation_resolver)
async def queue_call_end(reason: str) -> None:
logger.debug(f"文本测试收到会话结束请求: {reason}")
@@ -738,8 +744,6 @@ class TextPipelineRunner:
tool_executor_factory=create_test_executor,
),
)
executor = executor_holder["executor"]
# A transportless text run cannot synthesize audio, but the configured
# greeting has already completed before scripted inputs begin. Mirror
# production's provider-safe context fact so the model does not repeat it.
@@ -772,25 +776,11 @@ class TextPipelineRunner:
# response is therefore the equivalent of playback ending.
await call_end.finish()
all_tool_behaviors = [
behavior
for turn in definition.turns
for behavior in turn.behaviors
if behavior.type == "tool_call"
]
executor.set_turn_behaviors(all_tool_behaviors)
runner_task: asyncio.Task[None] | None = None
try:
startup_runtime_record_marker = len(capture.runtime_records)
simulation_resolver.enter_startup()
await brain.run_preflight()
if executor.unmocked_calls:
raise TestExecutionError(
code="UNMOCKED_TOOL_CALL",
message="启动阶段调用了未配置 Mock 的工具: "
+ "、".join(executor.unmocked_calls),
stage="tool",
)
await runner.add_workers(worker)
runner_task = asyncio.create_task(
@@ -803,12 +793,14 @@ class TextPipelineRunner:
marker = capture.window.activity_count
await brain.on_connected(greeting_pending=False)
await brain.on_client_ready()
simulation_resolver.raise_if_failed()
await _wait_for_startup(
worker,
capture,
runner_task,
marker,
)
simulation_resolver.raise_if_failed()
startup_greeting = _resolve_startup_greeting(
greeting,
capture.opening_speech_outputs,
@@ -844,7 +836,7 @@ class TextPipelineRunner:
worker,
runner_task,
capture,
executor,
simulation_resolver,
turn,
index,
image_assets=image_assets,
@@ -873,9 +865,24 @@ class TextPipelineRunner:
)
except TestExecutionError:
raise
except ToolSimulationError as exc:
raise TestExecutionError(
code=exc.code,
message=str(exc),
stage="tool",
retryable=False,
) from exc
except asyncio.CancelledError:
raise
except Exception as exc:
if simulation_resolver.failure is not None:
error = simulation_resolver.failure
raise TestExecutionError(
code=error.code,
message=str(error),
stage="tool",
retryable=False,
) from exc
raise TestExecutionError(
code="PIPELINE_EXECUTION_FAILED",
message=str(exc) or type(exc).__name__,
@@ -894,17 +901,14 @@ class TextPipelineRunner:
worker: PipelineWorker,
runner_task: asyncio.Task[None],
capture: _TextCaptureProcessor,
executor: MockToolExecutor,
simulation_resolver: ToolSimulationResolver,
turn: FixedInputTurn,
index: int,
*,
image_assets: dict[str, bytes],
workflow_vision_scope: dict[str, Any],
) -> RawTurnResult:
tool_behaviors = [
behavior for behavior in turn.behaviors if behavior.type == "tool_call"
]
executor.set_turn_behaviors(tool_behaviors)
simulation_resolver.enter_turn(turn.id)
activity_marker = capture.window.activity_count
output_marker = len(capture.outputs)
tool_marker = len(capture.tool_calls)
@@ -942,13 +946,7 @@ class TextPipelineRunner:
LLMMessagesAppendFrame(messages=[user_message], run_llm=True)
)
await _wait_for_quiescence(capture, runner_task, activity_marker)
if executor.unmocked_calls:
raise TestExecutionError(
code="UNMOCKED_TOOL_CALL",
message="本轮调用了未配置 Mock 的工具: "
+ "、".join(executor.unmocked_calls),
stage="tool",
)
simulation_resolver.raise_if_failed()
return RawTurnResult(
id=turn.id,
index=index,