wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(testing): add fixed tool simulations

Browse Source
This commit is contained in:
Xin Wang
2026-08-26 09:36:37 +08:00
parent 26582bfb87
commit 239b9027ca
16 changed files with 1525 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_debug_auto_test.py
+25 -9
View File
@@ -7,7 +7,11 @@ from pipecat.processors.aggregators.llm_context import LLMContext
from services.brains import BrainRuntime, build_brain
from services.brains.base import GREETING_CONTEXT_MARKER
from services.runtime_variables import DynamicVariableStore, prepare_dynamic_config
from services.test_runs.mock_tools import MockToolExecutor, TextClientToolPort
from services.test_runs.mock_tools import (
MockToolExecutor,
TextClientToolPort,
ToolSimulationResolver,
)
from services.test_runs.text_runner import (
_TextCaptureProcessor,
_opening_speech_from_message,
@@ -16,7 +20,7 @@ from services.test_runs.text_runner import (
_wait_for_startup,
_wait_for_quiescence,
)
from test_schemas import BatchRunCreate, BatchStartupResult
from test_schemas import BatchRunCreate, BatchStartupResult, FixedToolSimulation
class FakeLLM:
@@ -154,8 +158,24 @@ class DebugAutoTestContractTests(unittest.TestCase):
"message_started",
)
def test_text_client_port_records_automatic_message_confirmation(self):
port = TextClientToolPort()
def test_text_client_port_records_configured_message_confirmation(self):
resolver = ToolSimulationResolver(
[
FixedToolSimulation.model_validate(
{
"toolId": "tool_message",
"functionName": "show_message",
"mode": "fixed",
"response": {
"outcome": "success",
"body": '{"action":"confirmed"}',
"delayMs": 0,
},
}
)
]
)
port = TextClientToolPort(resolver)
result = asyncio.run(
port.call(
@@ -230,12 +250,9 @@ class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
brain = build_brain(cfg)
llm = FakeLLM()
call_end = FakeCallEnd()
executor_holder = {}
def create_executor(store):
executor = MockToolExecutor(store)
executor_holder["executor"] = executor
return executor
return MockToolExecutor(store, ToolSimulationResolver([]))
await brain.setup(
cfg,
@@ -256,7 +273,6 @@ class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase):
self.assertTrue(call_end.ending)
self.assertEqual(call_end.reason, "测试完成")
self.assertEqual(params.result["action"], "ending_call")
self.assertEqual(executor_holder["executor"].unmocked_calls, [])
async def test_text_capture_keeps_runtime_records_for_auto_test_ui(self):
event = {
backend/tests/test_startup_actions.py
+69 -1
View File
@@ -7,7 +7,11 @@ from fastapi import HTTPException
from routes.assistants import _validate_startup_actions
from models import RuntimeToolBinding
from schemas import AssistantUpsert
from services.config_resolver import _runtime_tool_ids
from services.config_resolver import (
_implicit_startup_tool_names,
_runtime_tool_ids,
_tools_for,
)
def startup_body(
@@ -57,6 +61,25 @@ class FakeSession:
return self.tool if self.tool is not None and tool_id == self.tool.id else None
class FakeToolQueryResult:
def __init__(self, tools):
self.tools = tools
def scalars(self):
return self
def all(self):
return self.tools
class FakeToolCatalogSession:
def __init__(self, tools):
self.tools = tools
async def execute(self, _statement):
return FakeToolQueryResult(self.tools)
class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase):
def test_runtime_pool_includes_bound_and_startup_only_tools(self):
self.assertEqual(
@@ -75,6 +98,51 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase):
["conversation_tool", "shared_tool", "opening_tool"],
)
def test_confirmation_opening_declares_implicit_show_message_tool(self):
self.assertEqual(
_implicit_startup_tool_names(
{
"opening_mode": "confirmation",
"opening_message": {
"title": "重要提示",
"message": "请确认已阅读。",
},
}
),
["show_message"],
)
self.assertEqual(
_implicit_startup_tool_names(
{
"openingMode": "playback",
"openingMessage": {"message": "无需确认"},
}
),
[],
)
async def test_implicit_show_message_enters_runtime_pool_without_binding(self):
tool = SimpleNamespace(
id="tool_show_message_default",
name="显示确认消息",
function_name="show_message",
type="client",
description="显示确认弹窗",
definition={"config": {}},
secrets={},
mcp_server_id=None,
)
runtime_tools = await _tools_for(
FakeToolCatalogSession([tool]),
SimpleNamespace(type="prompt"),
[],
["show_message"],
)
self.assertEqual([item.id for item in runtime_tools], [tool.id])
self.assertEqual(runtime_tools[0].function_name, "show_message")
def test_realtime_rejects_startup_actions(self):
with self.assertRaisesRegex(ValueError, "Realtime"):
AssistantUpsert(
backend/tests/test_tool_simulations.py
+312
View File
@@ -0,0 +1,312 @@
import unittest
from pydantic import ValidationError
from services.message_policy import MESSAGE_CONFIRMATION
from services.message_stage import (
MessageDisplaySpec,
MessageStageRunner,
MessageStageSpec,
)
from services.test_runs.mock_tools import (
TextClientToolPort,
ToolSimulationError,
ToolSimulationResolver,
)
from test_schemas import TestCaseDefinition
def definition_with_simulation(simulation: dict) -> TestCaseDefinition:
return TestCaseDefinition.model_validate(
{
"turns": [
{
"id": "turn_1",
"userInput": "查询订单",
"behaviors": [
{
"id": "behavior_1",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 1,
"maxCalls": 1,
"paramAssertions": [],
}
],
}
],
"toolSimulations": [simulation],
}
)
class ToolSimulationSchemaTests(unittest.TestCase):
def test_called_assertion_requires_simulation(self):
with self.assertRaisesRegex(ValidationError, "缺少工具模拟"):
TestCaseDefinition.model_validate(
{
"turns": [
{
"id": "turn_1",
"userInput": "查询订单",
"behaviors": [
{
"id": "behavior_1",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 1,
"paramAssertions": [],
}
],
}
]
}
)
def test_sequence_turn_scope_must_reference_existing_turn(self):
with self.assertRaisesRegex(ValidationError, "不存在的轮次"):
definition_with_simulation(
{
"toolId": "tool_order",
"functionName": "query_order",
"mode": "sequence",
"steps": [
{
"id": "step_1",
"scope": {"phase": "turn", "turnId": "missing"},
"outcome": "success",
"body": "{}",
"delayMs": 0,
}
],
}
)
def test_legacy_equal_responses_become_fixed_simulation(self):
definition = TestCaseDefinition.model_validate(
{
"turns": [
{
"id": "turn_1",
"userInput": "第一次查询",
"behaviors": [
{
"id": "behavior_1",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 1,
"paramAssertions": [],
"mockResponse": {
"outcome": "success",
"body": '{"orderId":"A1"}',
"delayMs": 0,
},
}
],
},
{
"id": "turn_2",
"userInput": "再次查询",
"behaviors": [
{
"id": "behavior_2",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 1,
"paramAssertions": [],
"mockResponse": {
"outcome": "success",
"body": '{"orderId":"A1"}',
"delayMs": 0,
},
}
],
},
]
}
)
self.assertEqual(definition.tool_simulations[0].mode, "fixed")
dumped = definition.model_dump(mode="json", by_alias=True)
self.assertNotIn("mockResponse", dumped["turns"][0]["behaviors"][0])
def test_legacy_different_responses_become_scoped_sequence(self):
definition = TestCaseDefinition.model_validate(
{
"turns": [
{
"id": "turn_1",
"userInput": "第一次查询",
"behaviors": [
{
"id": "behavior_1",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 2,
"paramAssertions": [],
"mockResponse": {
"outcome": "success",
"body": '{"orderId":"A1"}',
"delayMs": 0,
},
}
],
},
{
"id": "turn_2",
"userInput": "再次查询",
"behaviors": [
{
"id": "behavior_2",
"type": "tool_call",
"toolId": "tool_order",
"functionName": "query_order",
"expectation": "called",
"minCalls": 1,
"paramAssertions": [],
"mockResponse": {
"outcome": "error",
"body": '{"message":"not found"}',
"delayMs": 0,
},
}
],
},
]
}
)
simulation = definition.tool_simulations[0]
self.assertEqual(simulation.mode, "sequence")
self.assertEqual(len(simulation.steps), 3)
self.assertEqual(simulation.steps[0].scope.turn_id, "turn_1")
self.assertEqual(simulation.steps[2].scope.turn_id, "turn_2")
class ToolSimulationResolverTests(unittest.IsolatedAsyncioTestCase):
async def test_client_message_stage_keeps_unmocked_failure_for_runner(self):
resolver = ToolSimulationResolver([])
stage = MessageStageRunner(TextClientToolPort(resolver))
result = await stage.run(
MessageStageSpec(
display=MessageDisplaySpec(title="确认", message="请确认"),
completion_policy=MESSAGE_CONFIRMATION,
)
)
self.assertFalse(result.succeeded)
self.assertIn("UNMOCKED_TOOL_CALL", result.error or "")
with self.assertRaisesRegex(ToolSimulationError, "UNMOCKED_TOOL_CALL"):
resolver.raise_if_failed()
async def test_fixed_response_can_be_reused(self):
definition = definition_with_simulation(
{
"toolId": "tool_order",
"functionName": "query_order",
"mode": "fixed",
"response": {
"outcome": "success",
"body": '{"orderId":"A1"}',
"delayMs": 0,
},
}
)
resolver = ToolSimulationResolver(definition.tool_simulations)
resolver.enter_turn("turn_1")
first = await resolver.resolve(
tool_id="tool_order", function_name="query_order"
)
second = await resolver.resolve(
tool_id="tool_order", function_name="query_order"
)
self.assertEqual(first, {"orderId": "A1", "status": "ok"})
self.assertEqual(second, first)
async def test_sequence_consumes_client_choices_then_exhausts(self):
definition = definition_with_simulation(
{
"toolId": "tool_order",
"functionName": "query_order",
"mode": "sequence",
"steps": [
{
"id": "step_1",
"scope": {"phase": "turn", "turnId": "turn_1"},
"outcome": "success",
"body": '{"action":"selected","value":"basic"}',
"delayMs": 0,
},
{
"id": "step_2",
"scope": {"phase": "any"},
"outcome": "success",
"body": '{"action":"dismissed"}',
"delayMs": 0,
},
],
}
)
resolver = ToolSimulationResolver(definition.tool_simulations)
resolver.enter_turn("turn_1")
port = TextClientToolPort(resolver)
selected = await port.call("query_order", {})
dismissed = await port.call("query_order", {})
self.assertEqual(selected["data"]["value"], "basic")
self.assertEqual(dismissed["data"]["action"], "dismissed")
with self.assertRaisesRegex(
ToolSimulationError, "TOOL_SIMULATION_EXHAUSTED"
):
await port.call("query_order", {})
async def test_missing_and_scope_mismatch_are_distinct(self):
definition = definition_with_simulation(
{
"toolId": "tool_order",
"functionName": "query_order",
"mode": "sequence",
"steps": [
{
"id": "step_1",
"scope": {"phase": "turn", "turnId": "turn_1"},
"outcome": "success",
"body": "{}",
"delayMs": 0,
}
],
}
)
resolver = ToolSimulationResolver(definition.tool_simulations)
resolver.enter_startup()
with self.assertRaisesRegex(
ToolSimulationError, "TOOL_SIMULATION_SCOPE_MISMATCH"
):
await resolver.resolve(
tool_id="tool_order", function_name="query_order"
)
with self.assertRaisesRegex(
ToolSimulationError, "TOOL_SIMULATION_SCOPE_MISMATCH"
):
resolver.raise_if_failed()
with self.assertRaisesRegex(ToolSimulationError, "UNMOCKED_TOOL_CALL"):
await resolver.resolve(tool_id="missing", function_name="missing")
if __name__ == "__main__":
unittest.main()