wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: organize tools and enable qwen realtime calls

Browse Source
This commit is contained in:
Xin Wang
2026-08-14 00:15:00 +08:00
parent eb111a3890
commit 5be21f5ccc
71 changed files with 5771 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_prompt_realtime_tools.py
+605
View File
@@ -0,0 +1,605 @@
from __future__ import annotations
import asyncio
import json
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock
from fastapi import HTTPException
from models import AssistantConfig, RuntimeTool, RuntimeToolBinding
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
OutputTransportMessageUrgentFrame,
)
from services.brains.base import RealtimeBrainRuntime
from services.brains.prompt_realtime_tools import PromptRealtimeToolController
from services.pipecat.call_lifecycle import (
CallEndCoordinator,
FixedSpeechPlaybackMarkerFrame,
)
from services.pipecat.realtime_tools import (
RealtimeToolResult,
RealtimeToolSession,
)
from services.realtime.launcher import validate_runtime_requirements
from services.runtime_variables import DynamicVariableStore
from services.system_tools import RESERVED_TOOL_NAMES
from routes.assistants import _validate_prompt_realtime_tool_capabilities
from routes.model_registry import _ensure_realtime_prompt_tools_remain_supported
from schemas import AssistantUpsert
class RealtimeToolSessionTests(unittest.IsolatedAsyncioTestCase):
async def test_follow_up_priority_and_one_response_create(self):
events: list[dict] = []
suppressed = AsyncMock()
async def send_event(event: dict) -> None:
events.append(event)
session = RealtimeToolSession(send_event, suppress_response=suppressed)
async def dispatch(name, _arguments, _call_id):
return RealtimeToolResult(
{"name": name},
follow_up={"first": "none", "second": "suppress"}[name],
)
session.set_dispatcher(dispatch)
session.submit_call(
response_id="resp",
name="second",
call_id="call_2",
arguments={},
output_index=1,
)
session.submit_call(
response_id="resp",
name="first",
call_id="call_1",
arguments={},
output_index=0,
)
session.finish_response("resp")
await session.wait_pending()
self.assertEqual(
[event["item"]["call_id"] for event in events],
["call_1", "call_2"],
)
self.assertFalse(any(event["type"] == "response.create" for event in events))
suppressed.assert_awaited_once()
async def test_new_input_stops_unstarted_side_effect_and_follow_up(self):
events: list[dict] = []
first_started = asyncio.Event()
release_first = asyncio.Event()
dispatched: list[str] = []
async def send_event(event: dict) -> None:
events.append(event)
session = RealtimeToolSession(send_event)
async def dispatch(name, _arguments, _call_id):
dispatched.append(name)
if name == "first":
first_started.set()
await release_first.wait()
return RealtimeToolResult({"status": "ok"})
session.set_dispatcher(dispatch)
for index, name in enumerate(("first", "dangerous_second")):
session.submit_call(
response_id="resp",
name=name,
call_id=f"call_{index}",
arguments={},
output_index=index,
)
session.finish_response("resp")
await first_started.wait()
session.invalidate_pending_follow_ups()
release_first.set()
await session.wait_pending()
self.assertEqual(dispatched, ["first"])
self.assertFalse(any(event["type"] == "response.create" for event in events))
cancelled = json.loads(events[-1]["item"]["output"])
self.assertEqual(cancelled["status"], "cancelled")
async def test_close_cancels_active_background_call(self):
started = asyncio.Event()
cancelled = asyncio.Event()
session = RealtimeToolSession(AsyncMock())
async def dispatch(_name, _arguments, _call_id):
started.set()
try:
await asyncio.Future()
except asyncio.CancelledError:
cancelled.set()
raise
session.set_dispatcher(dispatch)
session.submit_call(
response_id="resp",
name="slow",
call_id="call_slow",
arguments={},
)
session.finish_response("resp")
await started.wait()
await session.shutdown()
self.assertTrue(cancelled.is_set())
class _FakeRealtime:
def __init__(self) -> None:
self.dispatcher = None
self.updated_sessions: list[tuple[str, list]] = []
self.instructions: list[str] = []
self.spoken: list[str] = []
def set_tool_dispatcher(self, dispatcher) -> None:
self.dispatcher = dispatcher
async def update_session(self, prompt: str, tools: list) -> None:
self.updated_sessions.append((prompt, tools))
async def update_instructions(self, prompt: str) -> None:
self.instructions.append(prompt)
async def speak_fixed(self, content: str, **_kwargs):
self.spoken.append(content)
future = asyncio.get_running_loop().create_future()
future.set_result(None)
return future
class _FakeExecutor:
def __init__(self, result: dict) -> None:
self.result = result
self.calls: list[tuple[str, dict]] = []
@staticmethod
def schema_parts(_tool, _binding):
return {"order_id": {"type": "string"}}, ["order_id"]
async def execute(self, tool, arguments, *, binding):
self.calls.append((tool.function_name, dict(arguments)))
return dict(self.result)
class PromptRealtimeToolControllerTests(unittest.IsolatedAsyncioTestCase):
def setUp(self) -> None:
self.frames: list = []
self.ended: list[str] = []
self.inputs: list[bool] = []
self.realtime = _FakeRealtime()
async def queue_frame(self, frame) -> None:
self.frames.append(frame)
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
await frame.completion.mark_played()
async def queue_end(self, reason: str) -> None:
self.ended.append(reason)
def runtime(self, call_end=None, client_tools=None) -> RealtimeBrainRuntime:
return RealtimeBrainRuntime(
realtime=self.realtime,
queue_frame=self.queue_frame,
call_end=call_end or CallEndCoordinator(self.queue_end),
session_id="conv_test",
client_tools=client_tools,
set_input_enabled=self.inputs.append,
)
async def controller(self, cfg: AssistantConfig, **runtime_kwargs):
store = DynamicVariableStore.from_config(cfg)
controller = PromptRealtimeToolController(
cfg=cfg,
store=store,
runtime=self.runtime(**runtime_kwargs),
)
await controller.start()
return controller, store
async def test_update_state_is_atomic_and_refreshes_prompt_after_event(self):
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
prompt="城市 {{city}} / 确认 {{confirmed}}",
dynamic_variable_definitions={
"city": {"type": "string"},
"confirmed": {"type": "boolean"},
},
dynamic_variables={"city": "北京", "confirmed": False},
built_in_tools={
"updateState": {
"updates": [
{
"variableName": "city",
"expression": {
"type": "llm",
"parameterName": "new_city",
"valueType": "string",
"description": "新城市",
},
},
{
"variableName": "confirmed",
"expression": {
"type": "llm",
"parameterName": "confirmed",
"valueType": "boolean",
"description": "是否确认",
},
},
]
}
},
)
controller, store = await self.controller(cfg)
failed = await controller.dispatch_tool(
"update_state",
{"new_city": "上海", "confirmed": "yes"},
"call_1",
)
self.assertEqual(failed.output["status"], "error")
self.assertEqual(store.public_values(), {"city": "北京", "confirmed": False})
succeeded = await controller.dispatch_tool(
"update_state",
{"new_city": "上海", "confirmed": True},
"call_2",
)
self.assertEqual(succeeded.output["status"], "success")
self.assertEqual(store.public_values(), {"city": "上海", "confirmed": True})
variable_event = next(
frame.message
for frame in self.frames
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "session-variables"
)
self.assertEqual(variable_event["changed"], ["city", "confirmed"])
self.assertEqual(self.realtime.instructions[-1], "城市 上海 / 确认 true")
async def test_external_tool_uses_executor_mapping_policy_and_input_gate(self):
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
prompt="状态 {{order_status}}",
dynamic_variable_definitions={"order_status": {"type": "string"}},
dynamic_variables={"order_status": "unknown"},
tools=[
RuntimeTool(
id="tool_order",
name="订单查询",
function_name="lookup_order",
type="http",
definition={
"type": "http",
"config": {
"allow_interruptions": False,
"execution_mode": "immediate",
},
},
)
],
tool_bindings=[
RuntimeToolBinding(
tool_id="tool_order",
argument_bindings={"order_id": {"source": "model"}},
)
],
)
controller, store = await self.controller(cfg)
store.values["order_status"] = "paid"
controller._executor = _FakeExecutor(
{"status": "ok", "data": {"status": "paid"}, "updated_variables": ["order_status"]}
)
result = await controller.dispatch_tool(
"lookup_order", {"order_id": "A-1"}, "call_1"
)
self.assertEqual(result.follow_up, "continue")
self.assertEqual(controller._executor.calls, [("lookup_order", {"order_id": "A-1"})])
self.assertEqual(self.inputs, [False, True])
self.assertEqual(self.realtime.instructions[-1], "状态 paid")
trace_events = [
frame.message.get("event")
for frame in self.frames
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "workflow-event"
]
self.assertIn("tool_started", trace_events)
self.assertIn("tool_completed", trace_events)
async def test_missing_client_capability_emits_tool_skipped(self):
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
tools=[
RuntimeTool(
id="tool_photo",
name="显示拍照按钮",
function_name="set_photo_button_visible",
type="client",
definition={"type": "client", "config": {"parameters": []}},
)
],
tool_bindings=[RuntimeToolBinding(tool_id="tool_photo")],
)
client_tools = SimpleNamespace(supports=lambda _name: False)
await self.controller(cfg, client_tools=client_tools)
self.assertEqual(self.realtime.updated_sessions[-1][1], [])
skipped = next(
frame.message
for frame in self.frames
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("event") == "tool_skipped"
)
self.assertEqual(skipped["reason"], "client_capability_missing")
async def test_handoff_is_idempotent_and_updates_system_state(self):
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
built_in_tools={"requestHumanHandoff": {}},
)
controller, store = await self.controller(cfg)
await controller.dispatch_tool("request_human_handoff", {"reason": "投诉"}, "1")
await controller.dispatch_tool("request_human_handoff", {"reason": "再次"}, "2")
handoff_events = [
frame.message
for frame in self.frames
if isinstance(frame, OutputTransportMessageUrgentFrame)
and frame.message.get("type") == "handoff-requested"
]
self.assertEqual(len(handoff_events), 1)
self.assertEqual(store.values["system__handoff_status"], "requested")
async def test_custom_end_waits_for_playback_marker_before_hangup(self):
call_end = CallEndCoordinator(self.queue_end)
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
built_in_tools={
"endConversation": {
"messageType": "custom",
"customMessage": "感谢来电，再见。",
"captureReason": True,
"allowEndInterruption": False,
}
},
)
controller, _store = await self.controller(cfg, call_end=call_end)
result = await controller.dispatch_tool(
"end_conversation", {"reason": "completed"}, "call_end"
)
self.assertEqual(result.follow_up, "suppress")
self.assertEqual(self.ended, [])
await result.after_output()
self.assertEqual(self.realtime.spoken, ["感谢来电，再见。"])
self.assertEqual(self.ended, ["completed"])
self.assertIn(False, self.inputs)
async def test_generated_end_is_interruptible_and_idempotent(self):
call_end = CallEndCoordinator(self.queue_end)
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
built_in_tools={
"endConversation": {
"messageType": "none",
"allowEndInterruption": True,
}
},
)
controller, _store = await self.controller(cfg, call_end=call_end)
first = await controller.dispatch_tool(
"end_conversation", {"reason": "done"}, "call_1"
)
second = await controller.dispatch_tool(
"end_conversation", {"reason": "done"}, "call_2"
)
self.assertEqual(first.follow_up, "continue")
self.assertEqual(second.follow_up, "suppress")
await controller.on_user_speech_started()
self.assertFalse(call_end.ending)
self.assertEqual(self.ended, [])
async def test_generated_end_waits_for_its_own_transport_speech(self):
call_end = CallEndCoordinator(self.queue_end)
# Simulate completed speech from an older response. The new response
# must reset this state before deciding whether it can hang up.
await call_end.observe(BotStartedSpeakingFrame())
await call_end.observe(BotStoppedSpeakingFrame())
cfg = AssistantConfig(
type="prompt",
runtimeMode="realtime",
realtime_interface_type="qwen-audio-realtime",
built_in_tools={"endConversation": {"messageType": "none"}},
)
controller, _store = await self.controller(cfg, call_end=call_end)
await controller.dispatch_tool(
"end_conversation", {"reason": "done"}, "call_1"
)
controller.on_assistant_text_start()
await controller.on_response_done("再见。", False)
self.assertEqual(self.ended, [])
await call_end.observe(BotStartedSpeakingFrame())
self.assertEqual(self.ended, [])
await call_end.observe(BotStoppedSpeakingFrame())
self.assertEqual(self.ended, ["done"])
class PromptRealtimeCapabilityTests(unittest.TestCase):
@staticmethod
def config(interface_type: str, **updates) -> AssistantConfig:
values = {
"type": "prompt",
"runtimeMode": "realtime",
"realtime_interface_type": interface_type,
"realtimeModel": "model",
"realtime_base_url": "wss://example.test/realtime",
"realtime_api_key": "secret",
}
values.update(updates)
return AssistantConfig(**values)
def test_qwen_accepts_supported_tools_and_rejects_skip_turn(self):
validate_runtime_requirements(
self.config(
"qwen-audio-realtime",
built_in_tools={"updateState": {"updates": []}},
)
)
with self.assertRaisesRegex(ValueError, "skipTurn"):
validate_runtime_requirements(
self.config(
"qwen-audio-realtime",
built_in_tools={"skipTurn": {}},
)
)
def test_stepfun_rejects_prompt_realtime_tools(self):
with self.assertRaisesRegex(ValueError, "endConversation"):
validate_runtime_requirements(
self.config(
"stepfun-realtime",
built_in_tools={"endConversation": {}},
)
)
def test_runtime_rejects_reserved_external_names(self):
self.assertIn("end_conversation", RESERVED_TOOL_NAMES)
with self.assertRaisesRegex(ValueError, "平台保留名"):
validate_runtime_requirements(
self.config(
"qwen-audio-realtime",
tools=[
RuntimeTool(
id="legacy",
name="旧外部工具",
function_name="end_conversation",
type="http",
)
],
tool_bindings=[RuntimeToolBinding(tool_id="legacy")],
)
)
class _ScalarRows:
def __init__(self, rows) -> None:
self._rows = rows
def scalars(self):
return self
def all(self):
return list(self._rows)
class PromptRealtimeApiGateTests(unittest.IsolatedAsyncioTestCase):
async def test_save_accepts_qwen_features_and_rejects_stepfun_or_skip_turn(self):
resource = SimpleNamespace(
enabled=True,
capability="Realtime",
interface_type="qwen-audio-realtime",
)
qwen = SimpleNamespace(
enabled=True,
field_schema={
"features": {
"realtimePromptTools": {
"externalTypes": ["http", "mcp", "client"],
"builtIns": [
"endConversation",
"updateState",
"requestHumanHandoff",
],
}
}
},
)
session = AsyncMock()
async def get(_model, key):
return resource if key == "model_qwen" else qwen
session.get.side_effect = get
supported = AssistantUpsert(
name="Qwen",
type="prompt",
runtimeMode="realtime",
modelResourceIds={"Realtime": "model_qwen"},
builtInTools={"updateState": {"updates": []}},
)
await _validate_prompt_realtime_tool_capabilities(session, supported)
skipped = AssistantUpsert(
name="Qwen",
type="prompt",
runtimeMode="realtime",
modelResourceIds={"Realtime": "model_qwen"},
builtInTools={"skipTurn": {"description": "silent"}},
)
with self.assertRaisesRegex(HTTPException, "skipTurn"):
await _validate_prompt_realtime_tool_capabilities(session, skipped)
resource.interface_type = "stepfun-realtime"
stepfun = SimpleNamespace(enabled=True, field_schema={"fields": []})
session.get.side_effect = lambda _model, key: (
resource if key == "model_qwen" else stepfun
)
with self.assertRaisesRegex(HTTPException, "updateState"):
await _validate_prompt_realtime_tool_capabilities(session, supported)
async def test_model_resource_cannot_drop_features_while_tools_are_bound(self):
assistant = SimpleNamespace(
id="asst_qwen",
name="Realtime 客服",
built_in_tools={"end_conversation": {"capture_reason": True}},
)
session = AsyncMock()
session.execute.return_value = _ScalarRows([assistant])
resource = SimpleNamespace(id="model_qwen", capability="Realtime")
unsupported_definition = SimpleNamespace(
capability="Realtime",
field_schema={"fields": []},
)
with self.assertRaisesRegex(HTTPException, "新接口不支持其内建工具"):
await _ensure_realtime_prompt_tools_remain_supported(
session,
resource,
unsupported_definition,
)
if __name__ == "__main__":
unittest.main()