from __future__ import annotations import asyncio import json import unittest from types import SimpleNamespace from unittest.mock import AsyncMock from fastapi import HTTPException from models import AssistantConfig, RuntimeTool, RuntimeToolBinding from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, OutputTransportMessageUrgentFrame, ) from services.brains.base import RealtimeBrainRuntime from services.brains.prompt_realtime_tools import PromptRealtimeToolController from services.pipecat.call_lifecycle import ( CallEndCoordinator, FixedSpeechPlaybackMarkerFrame, ) from services.pipecat.realtime_tools import ( RealtimeToolResult, RealtimeToolSession, ) from services.realtime.launcher import validate_runtime_requirements from services.runtime_variables import DynamicVariableStore from services.system_tools import RESERVED_TOOL_NAMES from routes.assistants import _validate_prompt_realtime_tool_capabilities from routes.model_registry import _ensure_realtime_prompt_tools_remain_supported from schemas import AssistantUpsert class RealtimeToolSessionTests(unittest.IsolatedAsyncioTestCase): async def test_follow_up_priority_and_one_response_create(self): events: list[dict] = [] suppressed = AsyncMock() async def send_event(event: dict) -> None: events.append(event) session = RealtimeToolSession(send_event, suppress_response=suppressed) async def dispatch(name, _arguments, _call_id): return RealtimeToolResult( {"name": name}, follow_up={"first": "none", "second": "suppress"}[name], ) session.set_dispatcher(dispatch) session.submit_call( response_id="resp", name="second", call_id="call_2", arguments={}, output_index=1, ) session.submit_call( response_id="resp", name="first", call_id="call_1", arguments={}, output_index=0, ) session.finish_response("resp") await session.wait_pending() self.assertEqual( [event["item"]["call_id"] for event in events], ["call_1", "call_2"], ) self.assertFalse(any(event["type"] == "response.create" for event in events)) suppressed.assert_awaited_once() async def test_new_input_stops_unstarted_side_effect_and_follow_up(self): events: list[dict] = [] first_started = asyncio.Event() release_first = asyncio.Event() dispatched: list[str] = [] async def send_event(event: dict) -> None: events.append(event) session = RealtimeToolSession(send_event) async def dispatch(name, _arguments, _call_id): dispatched.append(name) if name == "first": first_started.set() await release_first.wait() return RealtimeToolResult({"status": "ok"}) session.set_dispatcher(dispatch) for index, name in enumerate(("first", "dangerous_second")): session.submit_call( response_id="resp", name=name, call_id=f"call_{index}", arguments={}, output_index=index, ) session.finish_response("resp") await first_started.wait() session.invalidate_pending_follow_ups() release_first.set() await session.wait_pending() self.assertEqual(dispatched, ["first"]) self.assertFalse(any(event["type"] == "response.create" for event in events)) cancelled = json.loads(events[-1]["item"]["output"]) self.assertEqual(cancelled["status"], "cancelled") async def test_close_cancels_active_background_call(self): started = asyncio.Event() cancelled = asyncio.Event() session = RealtimeToolSession(AsyncMock()) async def dispatch(_name, _arguments, _call_id): started.set() try: await asyncio.Future() except asyncio.CancelledError: cancelled.set() raise session.set_dispatcher(dispatch) session.submit_call( response_id="resp", name="slow", call_id="call_slow", arguments={}, ) session.finish_response("resp") await started.wait() await session.shutdown() self.assertTrue(cancelled.is_set()) class _FakeRealtime: def __init__(self) -> None: self.dispatcher = None self.updated_sessions: list[tuple[str, list]] = [] self.instructions: list[str] = [] self.spoken: list[str] = [] def set_tool_dispatcher(self, dispatcher) -> None: self.dispatcher = dispatcher async def update_session(self, prompt: str, tools: list) -> None: self.updated_sessions.append((prompt, tools)) async def update_instructions(self, prompt: str) -> None: self.instructions.append(prompt) async def speak_fixed(self, content: str, **_kwargs): self.spoken.append(content) future = asyncio.get_running_loop().create_future() future.set_result(None) return future class _FakeExecutor: def __init__(self, result: dict) -> None: self.result = result self.calls: list[tuple[str, dict]] = [] @staticmethod def schema_parts(_tool, _binding): return {"order_id": {"type": "string"}}, ["order_id"] async def execute(self, tool, arguments, *, binding): self.calls.append((tool.function_name, dict(arguments))) return dict(self.result) class PromptRealtimeToolControllerTests(unittest.IsolatedAsyncioTestCase): def setUp(self) -> None: self.frames: list = [] self.ended: list[str] = [] self.inputs: list[bool] = [] self.realtime = _FakeRealtime() async def queue_frame(self, frame) -> None: self.frames.append(frame) if isinstance(frame, FixedSpeechPlaybackMarkerFrame): await frame.completion.mark_played() async def queue_end(self, reason: str) -> None: self.ended.append(reason) def runtime(self, call_end=None, client_tools=None) -> RealtimeBrainRuntime: return RealtimeBrainRuntime( realtime=self.realtime, queue_frame=self.queue_frame, call_end=call_end or CallEndCoordinator(self.queue_end), session_id="conv_test", client_tools=client_tools, set_input_enabled=self.inputs.append, ) async def controller(self, cfg: AssistantConfig, **runtime_kwargs): store = DynamicVariableStore.from_config(cfg) controller = PromptRealtimeToolController( cfg=cfg, store=store, runtime=self.runtime(**runtime_kwargs), ) await controller.start() return controller, store async def test_update_state_is_atomic_and_refreshes_prompt_after_event(self): cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", prompt="城市 {{city}} / 确认 {{confirmed}}", dynamic_variable_definitions={ "city": {"type": "string"}, "confirmed": {"type": "boolean"}, }, dynamic_variables={"city": "北京", "confirmed": False}, built_in_tools={ "updateState": { "updates": [ { "variableName": "city", "expression": { "type": "llm", "parameterName": "new_city", "valueType": "string", "description": "新城市", }, }, { "variableName": "confirmed", "expression": { "type": "llm", "parameterName": "confirmed", "valueType": "boolean", "description": "是否确认", }, }, ] } }, ) controller, store = await self.controller(cfg) failed = await controller.dispatch_tool( "update_state", {"new_city": "上海", "confirmed": "yes"}, "call_1", ) self.assertEqual(failed.output["status"], "error") self.assertEqual(store.public_values(), {"city": "北京", "confirmed": False}) succeeded = await controller.dispatch_tool( "update_state", {"new_city": "上海", "confirmed": True}, "call_2", ) self.assertEqual(succeeded.output["status"], "success") self.assertEqual(store.public_values(), {"city": "上海", "confirmed": True}) variable_event = next( frame.message for frame in self.frames if isinstance(frame, OutputTransportMessageUrgentFrame) and frame.message.get("type") == "session-variables" ) self.assertEqual(variable_event["changed"], ["city", "confirmed"]) self.assertEqual(self.realtime.instructions[-1], "城市 上海 / 确认 true") async def test_external_tool_uses_executor_mapping_policy_and_input_gate(self): cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", prompt="状态 {{order_status}}", dynamic_variable_definitions={"order_status": {"type": "string"}}, dynamic_variables={"order_status": "unknown"}, tools=[ RuntimeTool( id="tool_order", name="订单查询", function_name="lookup_order", type="http", definition={ "type": "http", "config": { "allow_interruptions": False, "execution_mode": "immediate", }, }, ) ], tool_bindings=[ RuntimeToolBinding( tool_id="tool_order", argument_bindings={"order_id": {"source": "model"}}, ) ], ) controller, store = await self.controller(cfg) store.values["order_status"] = "paid" controller._executor = _FakeExecutor( {"status": "ok", "data": {"status": "paid"}, "updated_variables": ["order_status"]} ) result = await controller.dispatch_tool( "lookup_order", {"order_id": "A-1"}, "call_1" ) self.assertEqual(result.follow_up, "continue") self.assertEqual(controller._executor.calls, [("lookup_order", {"order_id": "A-1"})]) self.assertEqual(self.inputs, [False, True]) self.assertEqual(self.realtime.instructions[-1], "状态 paid") trace_events = [ frame.message.get("event") for frame in self.frames if isinstance(frame, OutputTransportMessageUrgentFrame) and frame.message.get("type") == "workflow-event" ] self.assertIn("tool_started", trace_events) self.assertIn("tool_completed", trace_events) async def test_missing_client_capability_emits_tool_skipped(self): cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", tools=[ RuntimeTool( id="tool_photo", name="显示拍照按钮", function_name="set_photo_button_visible", type="client", definition={ "type": "client", "config": { "allow_interruptions": True, "execution_mode": "async", "wait_for_response": True, "response_wait_mode": "timeout", "parameters": [], }, }, ) ], tool_bindings=[RuntimeToolBinding(tool_id="tool_photo")], ) client_tools = SimpleNamespace(supports=lambda _name: False) await self.controller(cfg, client_tools=client_tools) self.assertEqual(self.realtime.updated_sessions[-1][1], []) skipped = next( frame.message for frame in self.frames if isinstance(frame, OutputTransportMessageUrgentFrame) and frame.message.get("event") == "tool_skipped" ) self.assertEqual(skipped["reason"], "client_capability_missing") async def test_handoff_is_idempotent_and_updates_system_state(self): cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", built_in_tools={"requestHumanHandoff": {}}, ) controller, store = await self.controller(cfg) await controller.dispatch_tool("request_human_handoff", {"reason": "投诉"}, "1") await controller.dispatch_tool("request_human_handoff", {"reason": "再次"}, "2") handoff_events = [ frame.message for frame in self.frames if isinstance(frame, OutputTransportMessageUrgentFrame) and frame.message.get("type") == "handoff-requested" ] self.assertEqual(len(handoff_events), 1) self.assertEqual(store.values["system__handoff_status"], "requested") async def test_custom_end_waits_for_playback_marker_before_hangup(self): call_end = CallEndCoordinator(self.queue_end) cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", built_in_tools={ "endConversation": { "messageType": "custom", "customMessage": "感谢来电，再见。", "captureReason": True, "allowEndInterruption": False, } }, ) controller, _store = await self.controller(cfg, call_end=call_end) result = await controller.dispatch_tool( "end_conversation", {"reason": "completed"}, "call_end" ) self.assertEqual(result.follow_up, "suppress") self.assertEqual(self.ended, []) await result.after_output() self.assertEqual(self.realtime.spoken, ["感谢来电，再见。"]) self.assertEqual(self.ended, ["completed"]) self.assertIn(False, self.inputs) async def test_generated_end_is_interruptible_and_idempotent(self): call_end = CallEndCoordinator(self.queue_end) cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", built_in_tools={ "endConversation": { "messageType": "none", "allowEndInterruption": True, } }, ) controller, _store = await self.controller(cfg, call_end=call_end) first = await controller.dispatch_tool( "end_conversation", {"reason": "done"}, "call_1" ) second = await controller.dispatch_tool( "end_conversation", {"reason": "done"}, "call_2" ) self.assertEqual(first.follow_up, "continue") self.assertEqual(second.follow_up, "suppress") await controller.on_user_speech_started() self.assertFalse(call_end.ending) self.assertEqual(self.ended, []) async def test_generated_end_waits_for_its_own_transport_speech(self): call_end = CallEndCoordinator(self.queue_end) # Simulate completed speech from an older response. The new response # must reset this state before deciding whether it can hang up. await call_end.observe(BotStartedSpeakingFrame()) await call_end.observe(BotStoppedSpeakingFrame()) cfg = AssistantConfig( type="prompt", runtimeMode="realtime", realtime_interface_type="qwen-audio-realtime", built_in_tools={"endConversation": {"messageType": "none"}}, ) controller, _store = await self.controller(cfg, call_end=call_end) await controller.dispatch_tool( "end_conversation", {"reason": "done"}, "call_1" ) controller.on_assistant_text_start() await controller.on_response_done("再见。", False) self.assertEqual(self.ended, []) await call_end.observe(BotStartedSpeakingFrame()) self.assertEqual(self.ended, []) await call_end.observe(BotStoppedSpeakingFrame()) self.assertEqual(self.ended, ["done"]) class PromptRealtimeCapabilityTests(unittest.TestCase): @staticmethod def config(interface_type: str, **updates) -> AssistantConfig: values = { "type": "prompt", "runtimeMode": "realtime", "realtime_interface_type": interface_type, "realtimeModel": "model", "realtime_base_url": "wss://example.test/realtime", "realtime_api_key": "secret", } values.update(updates) return AssistantConfig(**values) def test_qwen_accepts_supported_tools_and_rejects_skip_turn(self): validate_runtime_requirements( self.config( "qwen-audio-realtime", built_in_tools={"updateState": {"updates": []}}, ) ) with self.assertRaisesRegex(ValueError, "skipTurn"): validate_runtime_requirements( self.config( "qwen-audio-realtime", built_in_tools={"skipTurn": {}}, ) ) def test_stepfun_rejects_prompt_realtime_tools(self): with self.assertRaisesRegex(ValueError, "endConversation"): validate_runtime_requirements( self.config( "stepfun-realtime", built_in_tools={"endConversation": {}}, ) ) def test_runtime_rejects_reserved_external_names(self): self.assertIn("end_conversation", RESERVED_TOOL_NAMES) with self.assertRaisesRegex(ValueError, "平台保留名"): validate_runtime_requirements( self.config( "qwen-audio-realtime", tools=[ RuntimeTool( id="reserved_name_tool", name="保留名冲突工具", function_name="end_conversation", type="http", definition={ "type": "http", "config": { "allow_interruptions": True, "execution_mode": "immediate", }, }, ) ], tool_bindings=[RuntimeToolBinding(tool_id="reserved_name_tool")], ) ) class _ScalarRows: def __init__(self, rows) -> None: self._rows = rows def scalars(self): return self def all(self): return list(self._rows) class PromptRealtimeApiGateTests(unittest.IsolatedAsyncioTestCase): async def test_save_accepts_qwen_features_and_rejects_stepfun_or_skip_turn(self): resource = SimpleNamespace( enabled=True, capability="Realtime", interface_type="qwen-audio-realtime", ) qwen = SimpleNamespace( enabled=True, field_schema={ "features": { "realtimePromptTools": { "externalTypes": ["http", "mcp", "client"], "builtIns": [ "endConversation", "updateState", "requestHumanHandoff", ], } } }, ) session = AsyncMock() async def get(_model, key): return resource if key == "model_qwen" else qwen session.get.side_effect = get supported = AssistantUpsert( name="Qwen", type="prompt", runtimeMode="realtime", modelResourceIds={"Realtime": "model_qwen"}, builtInTools={"updateState": {"updates": []}}, ) await _validate_prompt_realtime_tool_capabilities(session, supported) skipped = AssistantUpsert( name="Qwen", type="prompt", runtimeMode="realtime", modelResourceIds={"Realtime": "model_qwen"}, builtInTools={"skipTurn": {"description": "silent"}}, ) with self.assertRaisesRegex(HTTPException, "skipTurn"): await _validate_prompt_realtime_tool_capabilities(session, skipped) resource.interface_type = "stepfun-realtime" stepfun = SimpleNamespace(enabled=True, field_schema={"fields": []}) session.get.side_effect = lambda _model, key: ( resource if key == "model_qwen" else stepfun ) with self.assertRaisesRegex(HTTPException, "updateState"): await _validate_prompt_realtime_tool_capabilities(session, supported) async def test_model_resource_cannot_drop_features_while_tools_are_bound(self): assistant = SimpleNamespace( id="asst_qwen", name="Realtime 客服", built_in_tools={"end_conversation": {"capture_reason": True}}, ) session = AsyncMock() session.execute.return_value = _ScalarRows([assistant]) resource = SimpleNamespace(id="model_qwen", capability="Realtime") unsupported_definition = SimpleNamespace( capability="Realtime", field_schema={"fields": []}, ) with self.assertRaisesRegex(HTTPException, "新接口不支持其内建工具"): await _ensure_realtime_prompt_tools_remain_supported( session, resource, unsupported_definition, ) if __name__ == "__main__": unittest.main()