622 lines
23 KiB
Python
622 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock
|
|
|
|
from fastapi import HTTPException
|
|
from models import AssistantConfig, RuntimeTool, RuntimeToolBinding
|
|
from pipecat.frames.frames import (
|
|
BotStartedSpeakingFrame,
|
|
BotStoppedSpeakingFrame,
|
|
OutputTransportMessageUrgentFrame,
|
|
)
|
|
from services.brains.base import RealtimeBrainRuntime
|
|
from services.brains.prompt_realtime_tools import PromptRealtimeToolController
|
|
from services.pipecat.call_lifecycle import (
|
|
CallEndCoordinator,
|
|
FixedSpeechPlaybackMarkerFrame,
|
|
)
|
|
from services.pipecat.realtime_tools import (
|
|
RealtimeToolResult,
|
|
RealtimeToolSession,
|
|
)
|
|
from services.realtime.launcher import validate_runtime_requirements
|
|
from services.runtime_variables import DynamicVariableStore
|
|
from services.system_tools import RESERVED_TOOL_NAMES
|
|
from routes.assistants import _validate_prompt_realtime_tool_capabilities
|
|
from routes.model_registry import _ensure_realtime_prompt_tools_remain_supported
|
|
from schemas import AssistantUpsert
|
|
|
|
|
|
class RealtimeToolSessionTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_follow_up_priority_and_one_response_create(self):
|
|
events: list[dict] = []
|
|
suppressed = AsyncMock()
|
|
|
|
async def send_event(event: dict) -> None:
|
|
events.append(event)
|
|
|
|
session = RealtimeToolSession(send_event, suppress_response=suppressed)
|
|
|
|
async def dispatch(name, _arguments, _call_id):
|
|
return RealtimeToolResult(
|
|
{"name": name},
|
|
follow_up={"first": "none", "second": "suppress"}[name],
|
|
)
|
|
|
|
session.set_dispatcher(dispatch)
|
|
session.submit_call(
|
|
response_id="resp",
|
|
name="second",
|
|
call_id="call_2",
|
|
arguments={},
|
|
output_index=1,
|
|
)
|
|
session.submit_call(
|
|
response_id="resp",
|
|
name="first",
|
|
call_id="call_1",
|
|
arguments={},
|
|
output_index=0,
|
|
)
|
|
session.finish_response("resp")
|
|
await session.wait_pending()
|
|
|
|
self.assertEqual(
|
|
[event["item"]["call_id"] for event in events],
|
|
["call_1", "call_2"],
|
|
)
|
|
self.assertFalse(any(event["type"] == "response.create" for event in events))
|
|
suppressed.assert_awaited_once()
|
|
|
|
async def test_new_input_stops_unstarted_side_effect_and_follow_up(self):
|
|
events: list[dict] = []
|
|
first_started = asyncio.Event()
|
|
release_first = asyncio.Event()
|
|
dispatched: list[str] = []
|
|
|
|
async def send_event(event: dict) -> None:
|
|
events.append(event)
|
|
|
|
session = RealtimeToolSession(send_event)
|
|
|
|
async def dispatch(name, _arguments, _call_id):
|
|
dispatched.append(name)
|
|
if name == "first":
|
|
first_started.set()
|
|
await release_first.wait()
|
|
return RealtimeToolResult({"status": "ok"})
|
|
|
|
session.set_dispatcher(dispatch)
|
|
for index, name in enumerate(("first", "dangerous_second")):
|
|
session.submit_call(
|
|
response_id="resp",
|
|
name=name,
|
|
call_id=f"call_{index}",
|
|
arguments={},
|
|
output_index=index,
|
|
)
|
|
session.finish_response("resp")
|
|
await first_started.wait()
|
|
session.invalidate_pending_follow_ups()
|
|
release_first.set()
|
|
await session.wait_pending()
|
|
|
|
self.assertEqual(dispatched, ["first"])
|
|
self.assertFalse(any(event["type"] == "response.create" for event in events))
|
|
cancelled = json.loads(events[-1]["item"]["output"])
|
|
self.assertEqual(cancelled["status"], "cancelled")
|
|
|
|
async def test_close_cancels_active_background_call(self):
|
|
started = asyncio.Event()
|
|
cancelled = asyncio.Event()
|
|
session = RealtimeToolSession(AsyncMock())
|
|
|
|
async def dispatch(_name, _arguments, _call_id):
|
|
started.set()
|
|
try:
|
|
await asyncio.Future()
|
|
except asyncio.CancelledError:
|
|
cancelled.set()
|
|
raise
|
|
|
|
session.set_dispatcher(dispatch)
|
|
session.submit_call(
|
|
response_id="resp",
|
|
name="slow",
|
|
call_id="call_slow",
|
|
arguments={},
|
|
)
|
|
session.finish_response("resp")
|
|
await started.wait()
|
|
await session.shutdown()
|
|
|
|
self.assertTrue(cancelled.is_set())
|
|
|
|
|
|
class _FakeRealtime:
|
|
def __init__(self) -> None:
|
|
self.dispatcher = None
|
|
self.updated_sessions: list[tuple[str, list]] = []
|
|
self.instructions: list[str] = []
|
|
self.spoken: list[str] = []
|
|
|
|
def set_tool_dispatcher(self, dispatcher) -> None:
|
|
self.dispatcher = dispatcher
|
|
|
|
async def update_session(self, prompt: str, tools: list) -> None:
|
|
self.updated_sessions.append((prompt, tools))
|
|
|
|
async def update_instructions(self, prompt: str) -> None:
|
|
self.instructions.append(prompt)
|
|
|
|
async def speak_fixed(self, content: str, **_kwargs):
|
|
self.spoken.append(content)
|
|
future = asyncio.get_running_loop().create_future()
|
|
future.set_result(None)
|
|
return future
|
|
|
|
|
|
class _FakeExecutor:
|
|
def __init__(self, result: dict) -> None:
|
|
self.result = result
|
|
self.calls: list[tuple[str, dict]] = []
|
|
|
|
@staticmethod
|
|
def schema_parts(_tool, _binding):
|
|
return {"order_id": {"type": "string"}}, ["order_id"]
|
|
|
|
async def execute(self, tool, arguments, *, binding):
|
|
self.calls.append((tool.function_name, dict(arguments)))
|
|
return dict(self.result)
|
|
|
|
|
|
class PromptRealtimeToolControllerTests(unittest.IsolatedAsyncioTestCase):
|
|
def setUp(self) -> None:
|
|
self.frames: list = []
|
|
self.ended: list[str] = []
|
|
self.inputs: list[bool] = []
|
|
self.realtime = _FakeRealtime()
|
|
|
|
async def queue_frame(self, frame) -> None:
|
|
self.frames.append(frame)
|
|
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
|
|
await frame.completion.mark_played()
|
|
|
|
async def queue_end(self, reason: str) -> None:
|
|
self.ended.append(reason)
|
|
|
|
def runtime(self, call_end=None, client_tools=None) -> RealtimeBrainRuntime:
|
|
return RealtimeBrainRuntime(
|
|
realtime=self.realtime,
|
|
queue_frame=self.queue_frame,
|
|
call_end=call_end or CallEndCoordinator(self.queue_end),
|
|
session_id="conv_test",
|
|
client_tools=client_tools,
|
|
set_input_enabled=self.inputs.append,
|
|
)
|
|
|
|
async def controller(self, cfg: AssistantConfig, **runtime_kwargs):
|
|
store = DynamicVariableStore.from_config(cfg)
|
|
controller = PromptRealtimeToolController(
|
|
cfg=cfg,
|
|
store=store,
|
|
runtime=self.runtime(**runtime_kwargs),
|
|
)
|
|
await controller.start()
|
|
return controller, store
|
|
|
|
async def test_update_state_is_atomic_and_refreshes_prompt_after_event(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
prompt="城市 {{city}} / 确认 {{confirmed}}",
|
|
dynamic_variable_definitions={
|
|
"city": {"type": "string"},
|
|
"confirmed": {"type": "boolean"},
|
|
},
|
|
dynamic_variables={"city": "北京", "confirmed": False},
|
|
built_in_tools={
|
|
"updateState": {
|
|
"updates": [
|
|
{
|
|
"variableName": "city",
|
|
"expression": {
|
|
"type": "llm",
|
|
"parameterName": "new_city",
|
|
"valueType": "string",
|
|
"description": "新城市",
|
|
},
|
|
},
|
|
{
|
|
"variableName": "confirmed",
|
|
"expression": {
|
|
"type": "llm",
|
|
"parameterName": "confirmed",
|
|
"valueType": "boolean",
|
|
"description": "是否确认",
|
|
},
|
|
},
|
|
]
|
|
}
|
|
},
|
|
)
|
|
controller, store = await self.controller(cfg)
|
|
|
|
failed = await controller.dispatch_tool(
|
|
"update_state",
|
|
{"new_city": "上海", "confirmed": "yes"},
|
|
"call_1",
|
|
)
|
|
self.assertEqual(failed.output["status"], "error")
|
|
self.assertEqual(store.public_values(), {"city": "北京", "confirmed": False})
|
|
|
|
succeeded = await controller.dispatch_tool(
|
|
"update_state",
|
|
{"new_city": "上海", "confirmed": True},
|
|
"call_2",
|
|
)
|
|
self.assertEqual(succeeded.output["status"], "success")
|
|
self.assertEqual(store.public_values(), {"city": "上海", "confirmed": True})
|
|
variable_event = next(
|
|
frame.message
|
|
for frame in self.frames
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
|
and frame.message.get("type") == "session-variables"
|
|
)
|
|
self.assertEqual(variable_event["changed"], ["city", "confirmed"])
|
|
self.assertEqual(self.realtime.instructions[-1], "城市 上海 / 确认 true")
|
|
|
|
async def test_external_tool_uses_executor_mapping_policy_and_input_gate(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
prompt="状态 {{order_status}}",
|
|
dynamic_variable_definitions={"order_status": {"type": "string"}},
|
|
dynamic_variables={"order_status": "unknown"},
|
|
tools=[
|
|
RuntimeTool(
|
|
id="tool_order",
|
|
name="订单查询",
|
|
function_name="lookup_order",
|
|
type="http",
|
|
definition={
|
|
"type": "http",
|
|
"config": {
|
|
"allow_interruptions": False,
|
|
"execution_mode": "immediate",
|
|
},
|
|
},
|
|
)
|
|
],
|
|
tool_bindings=[
|
|
RuntimeToolBinding(
|
|
tool_id="tool_order",
|
|
argument_bindings={"order_id": {"source": "model"}},
|
|
)
|
|
],
|
|
)
|
|
controller, store = await self.controller(cfg)
|
|
store.values["order_status"] = "paid"
|
|
controller._executor = _FakeExecutor(
|
|
{"status": "ok", "data": {"status": "paid"}, "updated_variables": ["order_status"]}
|
|
)
|
|
|
|
result = await controller.dispatch_tool(
|
|
"lookup_order", {"order_id": "A-1"}, "call_1"
|
|
)
|
|
|
|
self.assertEqual(result.follow_up, "continue")
|
|
self.assertEqual(controller._executor.calls, [("lookup_order", {"order_id": "A-1"})])
|
|
self.assertEqual(self.inputs, [False, True])
|
|
self.assertEqual(self.realtime.instructions[-1], "状态 paid")
|
|
trace_events = [
|
|
frame.message.get("event")
|
|
for frame in self.frames
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
|
and frame.message.get("type") == "workflow-event"
|
|
]
|
|
self.assertIn("tool_started", trace_events)
|
|
self.assertIn("tool_completed", trace_events)
|
|
|
|
async def test_missing_client_capability_emits_tool_skipped(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
tools=[
|
|
RuntimeTool(
|
|
id="tool_photo",
|
|
name="显示拍照按钮",
|
|
function_name="set_photo_button_visible",
|
|
type="client",
|
|
definition={
|
|
"type": "client",
|
|
"config": {
|
|
"allow_interruptions": True,
|
|
"execution_mode": "async",
|
|
"wait_for_response": True,
|
|
"response_wait_mode": "timeout",
|
|
"parameters": [],
|
|
},
|
|
},
|
|
)
|
|
],
|
|
tool_bindings=[RuntimeToolBinding(tool_id="tool_photo")],
|
|
)
|
|
client_tools = SimpleNamespace(supports=lambda _name: False)
|
|
await self.controller(cfg, client_tools=client_tools)
|
|
|
|
self.assertEqual(self.realtime.updated_sessions[-1][1], [])
|
|
skipped = next(
|
|
frame.message
|
|
for frame in self.frames
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
|
and frame.message.get("event") == "tool_skipped"
|
|
)
|
|
self.assertEqual(skipped["reason"], "client_capability_missing")
|
|
|
|
async def test_handoff_is_idempotent_and_updates_system_state(self):
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
built_in_tools={"requestHumanHandoff": {}},
|
|
)
|
|
controller, store = await self.controller(cfg)
|
|
|
|
await controller.dispatch_tool("request_human_handoff", {"reason": "投诉"}, "1")
|
|
await controller.dispatch_tool("request_human_handoff", {"reason": "再次"}, "2")
|
|
|
|
handoff_events = [
|
|
frame.message
|
|
for frame in self.frames
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
|
and frame.message.get("type") == "handoff-requested"
|
|
]
|
|
self.assertEqual(len(handoff_events), 1)
|
|
self.assertEqual(store.values["system__handoff_status"], "requested")
|
|
|
|
async def test_custom_end_waits_for_playback_marker_before_hangup(self):
|
|
call_end = CallEndCoordinator(self.queue_end)
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
built_in_tools={
|
|
"endConversation": {
|
|
"messageType": "custom",
|
|
"customMessage": "感谢来电，再见。",
|
|
"captureReason": True,
|
|
"allowEndInterruption": False,
|
|
}
|
|
},
|
|
)
|
|
controller, _store = await self.controller(cfg, call_end=call_end)
|
|
|
|
result = await controller.dispatch_tool(
|
|
"end_conversation", {"reason": "completed"}, "call_end"
|
|
)
|
|
self.assertEqual(result.follow_up, "suppress")
|
|
self.assertEqual(self.ended, [])
|
|
await result.after_output()
|
|
|
|
self.assertEqual(self.realtime.spoken, ["感谢来电，再见。"])
|
|
self.assertEqual(self.ended, ["completed"])
|
|
self.assertIn(False, self.inputs)
|
|
|
|
async def test_generated_end_is_interruptible_and_idempotent(self):
|
|
call_end = CallEndCoordinator(self.queue_end)
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
built_in_tools={
|
|
"endConversation": {
|
|
"messageType": "none",
|
|
"allowEndInterruption": True,
|
|
}
|
|
},
|
|
)
|
|
controller, _store = await self.controller(cfg, call_end=call_end)
|
|
|
|
first = await controller.dispatch_tool(
|
|
"end_conversation", {"reason": "done"}, "call_1"
|
|
)
|
|
second = await controller.dispatch_tool(
|
|
"end_conversation", {"reason": "done"}, "call_2"
|
|
)
|
|
self.assertEqual(first.follow_up, "continue")
|
|
self.assertEqual(second.follow_up, "suppress")
|
|
await controller.on_user_speech_started()
|
|
|
|
self.assertFalse(call_end.ending)
|
|
self.assertEqual(self.ended, [])
|
|
|
|
async def test_generated_end_waits_for_its_own_transport_speech(self):
|
|
call_end = CallEndCoordinator(self.queue_end)
|
|
# Simulate completed speech from an older response. The new response
|
|
# must reset this state before deciding whether it can hang up.
|
|
await call_end.observe(BotStartedSpeakingFrame())
|
|
await call_end.observe(BotStoppedSpeakingFrame())
|
|
cfg = AssistantConfig(
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
built_in_tools={"endConversation": {"messageType": "none"}},
|
|
)
|
|
controller, _store = await self.controller(cfg, call_end=call_end)
|
|
await controller.dispatch_tool(
|
|
"end_conversation", {"reason": "done"}, "call_1"
|
|
)
|
|
|
|
controller.on_assistant_text_start()
|
|
await controller.on_response_done("再见。", False)
|
|
self.assertEqual(self.ended, [])
|
|
|
|
await call_end.observe(BotStartedSpeakingFrame())
|
|
self.assertEqual(self.ended, [])
|
|
await call_end.observe(BotStoppedSpeakingFrame())
|
|
self.assertEqual(self.ended, ["done"])
|
|
|
|
|
|
class PromptRealtimeCapabilityTests(unittest.TestCase):
|
|
@staticmethod
|
|
def config(interface_type: str, **updates) -> AssistantConfig:
|
|
values = {
|
|
"type": "prompt",
|
|
"runtimeMode": "realtime",
|
|
"realtime_interface_type": interface_type,
|
|
"realtimeModel": "model",
|
|
"realtime_base_url": "wss://example.test/realtime",
|
|
"realtime_api_key": "secret",
|
|
}
|
|
values.update(updates)
|
|
return AssistantConfig(**values)
|
|
|
|
def test_qwen_accepts_supported_tools_and_rejects_skip_turn(self):
|
|
validate_runtime_requirements(
|
|
self.config(
|
|
"qwen-audio-realtime",
|
|
built_in_tools={"updateState": {"updates": []}},
|
|
)
|
|
)
|
|
with self.assertRaisesRegex(ValueError, "skipTurn"):
|
|
validate_runtime_requirements(
|
|
self.config(
|
|
"qwen-audio-realtime",
|
|
built_in_tools={"skipTurn": {}},
|
|
)
|
|
)
|
|
|
|
def test_stepfun_rejects_prompt_realtime_tools(self):
|
|
with self.assertRaisesRegex(ValueError, "endConversation"):
|
|
validate_runtime_requirements(
|
|
self.config(
|
|
"stepfun-realtime",
|
|
built_in_tools={"endConversation": {}},
|
|
)
|
|
)
|
|
|
|
def test_runtime_rejects_reserved_external_names(self):
|
|
self.assertIn("end_conversation", RESERVED_TOOL_NAMES)
|
|
with self.assertRaisesRegex(ValueError, "平台保留名"):
|
|
validate_runtime_requirements(
|
|
self.config(
|
|
"qwen-audio-realtime",
|
|
tools=[
|
|
RuntimeTool(
|
|
id="reserved_name_tool",
|
|
name="保留名冲突工具",
|
|
function_name="end_conversation",
|
|
type="http",
|
|
definition={
|
|
"type": "http",
|
|
"config": {
|
|
"allow_interruptions": True,
|
|
"execution_mode": "immediate",
|
|
},
|
|
},
|
|
)
|
|
],
|
|
tool_bindings=[RuntimeToolBinding(tool_id="reserved_name_tool")],
|
|
)
|
|
)
|
|
|
|
|
|
class _ScalarRows:
|
|
def __init__(self, rows) -> None:
|
|
self._rows = rows
|
|
|
|
def scalars(self):
|
|
return self
|
|
|
|
def all(self):
|
|
return list(self._rows)
|
|
|
|
|
|
class PromptRealtimeApiGateTests(unittest.IsolatedAsyncioTestCase):
|
|
async def test_save_accepts_qwen_features_and_rejects_stepfun_or_skip_turn(self):
|
|
resource = SimpleNamespace(
|
|
enabled=True,
|
|
capability="Realtime",
|
|
interface_type="qwen-audio-realtime",
|
|
)
|
|
qwen = SimpleNamespace(
|
|
enabled=True,
|
|
field_schema={
|
|
"features": {
|
|
"realtimePromptTools": {
|
|
"externalTypes": ["http", "mcp", "client"],
|
|
"builtIns": [
|
|
"endConversation",
|
|
"updateState",
|
|
"requestHumanHandoff",
|
|
],
|
|
}
|
|
}
|
|
},
|
|
)
|
|
session = AsyncMock()
|
|
|
|
async def get(_model, key):
|
|
return resource if key == "model_qwen" else qwen
|
|
|
|
session.get.side_effect = get
|
|
supported = AssistantUpsert(
|
|
name="Qwen",
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
modelResourceIds={"Realtime": "model_qwen"},
|
|
builtInTools={"updateState": {"updates": []}},
|
|
)
|
|
await _validate_prompt_realtime_tool_capabilities(session, supported)
|
|
|
|
skipped = AssistantUpsert(
|
|
name="Qwen",
|
|
type="prompt",
|
|
runtimeMode="realtime",
|
|
modelResourceIds={"Realtime": "model_qwen"},
|
|
builtInTools={"skipTurn": {"description": "silent"}},
|
|
)
|
|
with self.assertRaisesRegex(HTTPException, "skipTurn"):
|
|
await _validate_prompt_realtime_tool_capabilities(session, skipped)
|
|
|
|
resource.interface_type = "stepfun-realtime"
|
|
stepfun = SimpleNamespace(enabled=True, field_schema={"fields": []})
|
|
session.get.side_effect = lambda _model, key: (
|
|
resource if key == "model_qwen" else stepfun
|
|
)
|
|
with self.assertRaisesRegex(HTTPException, "updateState"):
|
|
await _validate_prompt_realtime_tool_capabilities(session, supported)
|
|
|
|
async def test_model_resource_cannot_drop_features_while_tools_are_bound(self):
|
|
assistant = SimpleNamespace(
|
|
id="asst_qwen",
|
|
name="Realtime 客服",
|
|
built_in_tools={"end_conversation": {"capture_reason": True}},
|
|
)
|
|
session = AsyncMock()
|
|
session.execute.return_value = _ScalarRows([assistant])
|
|
resource = SimpleNamespace(id="model_qwen", capability="Realtime")
|
|
unsupported_definition = SimpleNamespace(
|
|
capability="Realtime",
|
|
field_schema={"fields": []},
|
|
)
|
|
|
|
with self.assertRaisesRegex(HTTPException, "新接口不支持其内建工具"):
|
|
await _ensure_realtime_prompt_tools_remain_supported(
|
|
session,
|
|
resource,
|
|
unsupported_definition,
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|