feat(testing): add fixed tool simulations
This commit is contained in:
Xin Wang
@@ -0,0 +1,312 @@
|
||||
import unittest
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from services.message_policy import MESSAGE_CONFIRMATION
|
||||
from services.message_stage import (
|
||||
MessageDisplaySpec,
|
||||
MessageStageRunner,
|
||||
MessageStageSpec,
|
||||
)
|
||||
from services.test_runs.mock_tools import (
|
||||
TextClientToolPort,
|
||||
ToolSimulationError,
|
||||
ToolSimulationResolver,
|
||||
)
|
||||
from test_schemas import TestCaseDefinition
|
||||
|
||||
|
||||
def definition_with_simulation(simulation: dict) -> TestCaseDefinition:
|
||||
return TestCaseDefinition.model_validate(
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"id": "turn_1",
|
||||
"userInput": "查询订单",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_1",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 1,
|
||||
"maxCalls": 1,
|
||||
"paramAssertions": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"toolSimulations": [simulation],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class ToolSimulationSchemaTests(unittest.TestCase):
|
||||
def test_called_assertion_requires_simulation(self):
|
||||
with self.assertRaisesRegex(ValidationError, "缺少工具模拟"):
|
||||
TestCaseDefinition.model_validate(
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"id": "turn_1",
|
||||
"userInput": "查询订单",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_1",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 1,
|
||||
"paramAssertions": [],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
def test_sequence_turn_scope_must_reference_existing_turn(self):
|
||||
with self.assertRaisesRegex(ValidationError, "不存在的轮次"):
|
||||
definition_with_simulation(
|
||||
{
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"mode": "sequence",
|
||||
"steps": [
|
||||
{
|
||||
"id": "step_1",
|
||||
"scope": {"phase": "turn", "turnId": "missing"},
|
||||
"outcome": "success",
|
||||
"body": "{}",
|
||||
"delayMs": 0,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
def test_legacy_equal_responses_become_fixed_simulation(self):
|
||||
definition = TestCaseDefinition.model_validate(
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"id": "turn_1",
|
||||
"userInput": "第一次查询",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_1",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 1,
|
||||
"paramAssertions": [],
|
||||
"mockResponse": {
|
||||
"outcome": "success",
|
||||
"body": '{"orderId":"A1"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "turn_2",
|
||||
"userInput": "再次查询",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_2",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 1,
|
||||
"paramAssertions": [],
|
||||
"mockResponse": {
|
||||
"outcome": "success",
|
||||
"body": '{"orderId":"A1"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(definition.tool_simulations[0].mode, "fixed")
|
||||
dumped = definition.model_dump(mode="json", by_alias=True)
|
||||
self.assertNotIn("mockResponse", dumped["turns"][0]["behaviors"][0])
|
||||
|
||||
def test_legacy_different_responses_become_scoped_sequence(self):
|
||||
definition = TestCaseDefinition.model_validate(
|
||||
{
|
||||
"turns": [
|
||||
{
|
||||
"id": "turn_1",
|
||||
"userInput": "第一次查询",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_1",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 2,
|
||||
"paramAssertions": [],
|
||||
"mockResponse": {
|
||||
"outcome": "success",
|
||||
"body": '{"orderId":"A1"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "turn_2",
|
||||
"userInput": "再次查询",
|
||||
"behaviors": [
|
||||
{
|
||||
"id": "behavior_2",
|
||||
"type": "tool_call",
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"expectation": "called",
|
||||
"minCalls": 1,
|
||||
"paramAssertions": [],
|
||||
"mockResponse": {
|
||||
"outcome": "error",
|
||||
"body": '{"message":"not found"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
simulation = definition.tool_simulations[0]
|
||||
self.assertEqual(simulation.mode, "sequence")
|
||||
self.assertEqual(len(simulation.steps), 3)
|
||||
self.assertEqual(simulation.steps[0].scope.turn_id, "turn_1")
|
||||
self.assertEqual(simulation.steps[2].scope.turn_id, "turn_2")
|
||||
|
||||
|
||||
class ToolSimulationResolverTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_client_message_stage_keeps_unmocked_failure_for_runner(self):
|
||||
resolver = ToolSimulationResolver([])
|
||||
stage = MessageStageRunner(TextClientToolPort(resolver))
|
||||
|
||||
result = await stage.run(
|
||||
MessageStageSpec(
|
||||
display=MessageDisplaySpec(title="确认", message="请确认"),
|
||||
completion_policy=MESSAGE_CONFIRMATION,
|
||||
)
|
||||
)
|
||||
|
||||
self.assertFalse(result.succeeded)
|
||||
self.assertIn("UNMOCKED_TOOL_CALL", result.error or "")
|
||||
with self.assertRaisesRegex(ToolSimulationError, "UNMOCKED_TOOL_CALL"):
|
||||
resolver.raise_if_failed()
|
||||
|
||||
async def test_fixed_response_can_be_reused(self):
|
||||
definition = definition_with_simulation(
|
||||
{
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"mode": "fixed",
|
||||
"response": {
|
||||
"outcome": "success",
|
||||
"body": '{"orderId":"A1"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
}
|
||||
)
|
||||
resolver = ToolSimulationResolver(definition.tool_simulations)
|
||||
resolver.enter_turn("turn_1")
|
||||
|
||||
first = await resolver.resolve(
|
||||
tool_id="tool_order", function_name="query_order"
|
||||
)
|
||||
second = await resolver.resolve(
|
||||
tool_id="tool_order", function_name="query_order"
|
||||
)
|
||||
|
||||
self.assertEqual(first, {"orderId": "A1", "status": "ok"})
|
||||
self.assertEqual(second, first)
|
||||
|
||||
async def test_sequence_consumes_client_choices_then_exhausts(self):
|
||||
definition = definition_with_simulation(
|
||||
{
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"mode": "sequence",
|
||||
"steps": [
|
||||
{
|
||||
"id": "step_1",
|
||||
"scope": {"phase": "turn", "turnId": "turn_1"},
|
||||
"outcome": "success",
|
||||
"body": '{"action":"selected","value":"basic"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
{
|
||||
"id": "step_2",
|
||||
"scope": {"phase": "any"},
|
||||
"outcome": "success",
|
||||
"body": '{"action":"dismissed"}',
|
||||
"delayMs": 0,
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
resolver = ToolSimulationResolver(definition.tool_simulations)
|
||||
resolver.enter_turn("turn_1")
|
||||
port = TextClientToolPort(resolver)
|
||||
|
||||
selected = await port.call("query_order", {})
|
||||
dismissed = await port.call("query_order", {})
|
||||
|
||||
self.assertEqual(selected["data"]["value"], "basic")
|
||||
self.assertEqual(dismissed["data"]["action"], "dismissed")
|
||||
with self.assertRaisesRegex(
|
||||
ToolSimulationError, "TOOL_SIMULATION_EXHAUSTED"
|
||||
):
|
||||
await port.call("query_order", {})
|
||||
|
||||
async def test_missing_and_scope_mismatch_are_distinct(self):
|
||||
definition = definition_with_simulation(
|
||||
{
|
||||
"toolId": "tool_order",
|
||||
"functionName": "query_order",
|
||||
"mode": "sequence",
|
||||
"steps": [
|
||||
{
|
||||
"id": "step_1",
|
||||
"scope": {"phase": "turn", "turnId": "turn_1"},
|
||||
"outcome": "success",
|
||||
"body": "{}",
|
||||
"delayMs": 0,
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
resolver = ToolSimulationResolver(definition.tool_simulations)
|
||||
resolver.enter_startup()
|
||||
|
||||
with self.assertRaisesRegex(
|
||||
ToolSimulationError, "TOOL_SIMULATION_SCOPE_MISMATCH"
|
||||
):
|
||||
await resolver.resolve(
|
||||
tool_id="tool_order", function_name="query_order"
|
||||
)
|
||||
with self.assertRaisesRegex(
|
||||
ToolSimulationError, "TOOL_SIMULATION_SCOPE_MISMATCH"
|
||||
):
|
||||
resolver.raise_if_failed()
|
||||
with self.assertRaisesRegex(ToolSimulationError, "UNMOCKED_TOOL_CALL"):
|
||||
await resolver.resolve(tool_id="missing", function_name="missing")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user