wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: organize tools and enable qwen realtime calls

Browse Source
This commit is contained in:
Xin Wang
2026-08-14 00:15:00 +08:00
parent eb111a3890
commit 5be21f5ccc
71 changed files with 5771 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_brains.py
+198 -149
View File
@@ -5,7 +5,7 @@ import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from models import AssistantConfig, RuntimeTool
from models import AssistantConfig, RuntimeTool, RuntimeToolBinding
from pipecat.flows import FlowManager
from pipecat.frames.frames import (
LLMContextFrame,
@@ -21,7 +21,7 @@ from pipecat.frames.frames import (
)
from pipecat.processors.aggregators.llm_context import LLMContext
from pipecat.processors.frame_processor import FrameDirection
from schemas import AssistantUpsert, REALTIME_CAPABLE_TYPES, SystemToolConfig
from schemas import AssistantUpsert, BuiltInTools, REALTIME_CAPABLE_TYPES
from services.brains import BrainRuntime, SPECS, build_brain
from services.brains.base import GREETING_CONTEXT_MARKER
from services.brains.dify_llm import (
@@ -130,6 +130,21 @@ def system_runtime_tool(kind: str, **config) -> RuntimeTool:
)
def built_in_config(kind: str, **config) -> dict:
key = {
"end_conversation": "end_conversation",
"update_state": "update_state",
"skip_turn": "skip_turn",
"request_human_handoff": "request_human_handoff",
}[kind]
base = {"description": f"{kind} system tool", **config}
return {key: base}
def merge_builtins(*configs: dict) -> dict:
return {key: value for config in configs for key, value in config.items()}
class BrainRegistryTests(unittest.TestCase):
def test_capability_matrix(self):
self.assertEqual(
@@ -139,7 +154,7 @@ class BrainRegistryTests(unittest.TestCase):
},
{
"prompt": frozenset({"pipeline", "realtime"}),
"workflow": frozenset({"pipeline"}),
"workflow": frozenset({"pipeline", "realtime"}),
"dify": frozenset({"pipeline"}),
"fastgpt": frozenset({"pipeline"}),
},
@@ -157,13 +172,13 @@ class BrainRegistryTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, "尚未实现"):
build_brain(AssistantConfig(type="opencode"))
def test_workflow_realtime_is_rejected_at_schema_boundary(self):
with self.assertRaises(ValueError):
AssistantUpsert(
name="workflow",
type="workflow",
runtimeMode="realtime",
)
def test_workflow_realtime_is_accepted_at_schema_boundary(self):
assistant = AssistantUpsert(
name="workflow",
type="workflow",
runtimeMode="realtime",
)
self.assertEqual(assistant.runtime_mode, "realtime")
def test_prompt_realtime_keeps_dynamic_variable_definitions(self):
assistant = AssistantUpsert(
@@ -180,36 +195,27 @@ class BrainRegistryTests(unittest.TestCase):
)
self.assertIn("user_name", assistant.dynamic_variable_definitions)
def test_system_tool_config_accepts_all_platform_actions(self):
for kind in (
"end_conversation",
"update_state",
"skip_turn",
"request_human_handoff",
):
self.assertEqual(SystemToolConfig(kind=kind).kind, kind)
def test_end_interruption_is_opt_in(self):
self.assertFalse(SystemToolConfig().allow_end_interruption)
self.assertTrue(
SystemToolConfig(allowEndInterruption=True).allow_end_interruption
def test_built_in_tools_are_typed_platform_config(self):
tools = BuiltInTools(
endConversation={"allowEndInterruption": True},
updateState={"updates": []},
skipTurn={},
requestHumanHandoff={},
)
def test_system_tool_config_rejects_unknown_kind(self):
with self.assertRaises(ValueError):
SystemToolConfig(kind="magic")
self.assertTrue(tools.end_conversation.allow_end_interruption)
self.assertEqual(tools.update_state.updates, [])
def test_workflow_keeps_dynamic_variables_and_tool_bindings(self):
assistant = AssistantUpsert(
name="workflow",
type="workflow",
toolIds=["tool_a"],
toolBindings=[{"toolId": "tool_a"}],
dynamicVariableDefinitions={
"customer": {"type": "string", "required": False, "default": "王先生"}
},
graph={},
)
self.assertEqual(assistant.tool_ids, ["tool_a"])
self.assertEqual(assistant.tool_bindings, [])
self.assertIn("customer", assistant.dynamic_variable_definitions)
@@ -310,13 +316,16 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
cfg = AssistantConfig(
type="prompt",
tools=[startup_tool, conversation_tool],
llm_tool_ids=[conversation_tool.id],
tool_bindings=[RuntimeToolBinding(tool_id=conversation_tool.id)],
startup={
"actions": [
{
"id": "opening_message",
"phase": "opening",
"tool_id": startup_tool.id,
"toolBinding": {
"toolId": startup_tool.id,
"argumentBindings": {},
},
"required": True,
}
]
@@ -361,7 +370,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
{
"id": tool.id,
"phase": "preflight",
"tool_id": tool.id,
"toolBinding": {
"toolId": tool.id,
"argumentBindings": {},
},
"required": True,
}
for tool in tools
@@ -422,8 +434,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
{
"id": "opening_data",
"phase": "opening",
"tool_id": "opening_data",
"arguments": {},
"toolBinding": {
"toolId": "opening_data",
"argumentBindings": {},
},
"required": True,
}
],
@@ -848,26 +862,13 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(queued_frames, [])
async def test_end_conversation_system_tool_is_owned_by_prompt_brain(self):
brain = build_brain(
AssistantConfig(
type="prompt",
tools=[
RuntimeTool(
id="end-call",
name="结束通话",
function_name="end_call",
type="system",
definition={
"config": {
"kind": "end_conversation",
"message_type": "none",
"capture_reason": True,
}
},
)
],
)
cfg = AssistantConfig(
type="prompt",
built_in_tools=built_in_config(
"end_conversation", capture_reason=True
),
)
brain = build_brain(cfg)
llm = FakeLLM()
call_end = FakeCallEnd()
visible_tools = []
@@ -876,23 +877,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
pass
await brain.setup(
AssistantConfig(
type="prompt",
tools=[
RuntimeTool(
id="end-call",
name="结束通话",
function_name="end_call",
type="system",
definition={
"config": {
"kind": "end_conversation",
"capture_reason": True,
}
},
)
],
),
cfg,
BrainRuntime(
context=LLMContext(messages=[]),
llm=llm,
@@ -903,10 +888,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
),
)
self.assertEqual(visible_tools[0].name, "end_call")
self.assertEqual(visible_tools[0].name, "end_conversation")
params = FakeFunctionParams({"reason": "用户已完成咨询"})
await brain.on_assistant_text_start("closing-turn")
await llm.functions["end_call"](params)
await llm.functions["end_conversation"](params)
self.assertEqual(call_end.reason, "用户已完成咨询")
self.assertFalse(call_end.finished)
await brain.on_assistant_text_end("closing-turn", "", False)
@@ -916,19 +901,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(params.result["action"], "ending_call")
async def test_system_end_conversation_waits_for_generated_closing_speech(self):
tool = RuntimeTool(
id="end-call",
name="结束通话",
function_name="end_call",
type="system",
definition={
"config": {
"kind": "end_conversation",
"message_type": "none",
}
},
cfg = AssistantConfig(
type="prompt",
built_in_tools=built_in_config("end_conversation"),
)
cfg = AssistantConfig(type="prompt", tools=[tool])
brain = build_brain(cfg)
llm = FakeLLM()
call_end = FakeCallEnd()
@@ -945,7 +921,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
),
)
await brain.on_assistant_text_start("closing-turn")
await llm.functions["end_call"](FakeFunctionParams({}))
await llm.functions["end_conversation"](FakeFunctionParams({}))
await brain.on_assistant_text_end(
"closing-turn",
"感谢您的来电。祝您生活愉快，再见。",
@@ -966,12 +942,23 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
"default": None,
}
},
tools=[
system_runtime_tool("end_conversation"),
system_runtime_tool("update_state"),
system_runtime_tool("skip_turn"),
system_runtime_tool("request_human_handoff"),
],
built_in_tools=merge_builtins(
built_in_config("end_conversation"),
built_in_config(
"update_state",
updates=[{
"variableName": "user_name",
"expression": {
"type": "llm",
"parameterName": "user_name",
"valueType": "string",
"description": "更新动态变量 user_name。",
},
}],
),
built_in_config("skip_turn"),
built_in_config("request_human_handoff"),
),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1021,7 +1008,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_end_conversation_ends_call_after_generated_speech(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("end_conversation")],
built_in_tools=built_in_config("end_conversation"),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1059,12 +1046,9 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_end_conversation_passes_interruptible_tool_setting(self):
cfg = AssistantConfig(
type="prompt",
tools=[
system_runtime_tool(
"end_conversation",
allow_end_interruption=True,
)
],
built_in_tools=built_in_config(
"end_conversation", allow_end_interruption=True
),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1089,7 +1073,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_end_conversation_finishes_when_no_speech(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("end_conversation")],
built_in_tools=built_in_config("end_conversation"),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1119,7 +1103,18 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
dynamic_variable_definitions={
"user_name": {"type": "string", "required": False, "default": None}
},
tools=[system_runtime_tool("update_state")],
built_in_tools=built_in_config(
"update_state",
updates=[{
"variableName": "user_name",
"expression": {
"type": "llm",
"parameterName": "user_name",
"valueType": "string",
"description": "更新用户名",
},
}],
),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1160,10 +1155,24 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
)
self.assertTrue(prompts[-1].endswith("当前用户名: 王小明"))
async def test_update_state_rejects_undeclared_variable(self):
async def test_update_state_rejects_missing_configured_value(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("update_state")],
dynamic_variable_definitions={
"known_key": {"type": "string", "required": False, "default": None}
},
built_in_tools=built_in_config(
"update_state",
updates=[{
"variableName": "known_key",
"expression": {
"type": "llm",
"parameterName": "known_key",
"valueType": "string",
"description": "更新已知值",
},
}],
),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1183,12 +1192,12 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
params = FakeFunctionParams({"unknown_key": "x"})
await llm.functions["update_state"](params)
self.assertEqual(params.result["status"], "error")
self.assertIn("未声明", params.result["message"])
self.assertIn("缺少", params.result["message"])
async def test_skip_turn_suppresses_response(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("skip_turn")],
built_in_tools=built_in_config("skip_turn"),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1214,7 +1223,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
async def test_request_human_handoff_keeps_call_available(self):
cfg = AssistantConfig(
type="prompt",
tools=[system_runtime_tool("request_human_handoff")],
built_in_tools=built_in_config("request_human_handoff"),
)
brain = build_brain(cfg)
llm = FakeLLM()
@@ -1320,14 +1329,23 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase):
"required": False,
},
],
"dynamic_variable_assignments": {
"order_status": "response.order.status"
},
}
},
secrets={"dynamic_variables": {"secret__token": "server-token"}},
)
],
tool_bindings=[
RuntimeToolBinding(
tool_id="lookup",
argument_bindings={
"order_id": {"source": "model"},
"Authorization": {"source": "model"},
},
result_assignments={
"order_status": "response.order.status"
},
)
],
),
{},
assistant_id="asst_1",
@@ -1380,12 +1398,23 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"type": "agent",
"data": {
"inheritGlobalConfig": False,
"toolIds": [
"tool_update_state",
"tool_skip_turn",
"tool_request_human_handoff",
],
"stateVariableNames": ["customer_name"],
"toolBindings": [],
"builtInTools": merge_builtins(
built_in_config(
"update_state",
updates=[{
"variableName": "customer_name",
"expression": {
"type": "llm",
"parameterName": "customer_name",
"valueType": "string",
"description": "更新客户姓名",
},
}],
),
built_in_config("skip_turn"),
built_in_config("request_human_handoff"),
),
},
},
],
@@ -1403,11 +1432,6 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"default": None,
},
},
tools=[
system_runtime_tool("update_state"),
system_runtime_tool("skip_turn"),
system_runtime_tool("request_human_handoff"),
],
),
{},
assistant_id="asst_workflow_system_tools",
@@ -1435,6 +1459,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
)
brain._emit_variables = AsyncMock()
brain._refresh_agent_prompt = AsyncMock()
brain._state.enter("agent", WorkflowStatus.WAITING_USER)
node_config = brain._agent_config("agent")
functions = {tool.name: tool for tool in node_config["functions"]}
@@ -1465,7 +1490,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
None,
)
self.assertEqual(unauthorized["status"], "error")
self.assertIn("未获当前节点授权", unauthorized["message"])
self.assertIn("缺少", unauthorized["message"])
handoff = await functions["request_human_handoff"].handler(
{"reason": "用户要求人工"},
@@ -1494,10 +1519,19 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"id": "set_state",
"type": "update_state",
"data": {
"assignments": {
"confirmed": True,
"display_name": "{{source_name}}",
}
"updates": [
{
"variableName": "confirmed",
"expression": {"type": "constant", "value": True},
},
{
"variableName": "display_name",
"expression": {
"type": "variable",
"variableName": "source_name",
},
},
]
},
},
],
@@ -1640,8 +1674,15 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
},
)
self.assertEqual(brain._flow_tool(timeout_tool, "start").timeout_secs, 7.0)
self.assertIsNone(brain._flow_tool(session_tool, "start").timeout_secs)
timeout_binding = RuntimeToolBinding(tool_id=timeout_tool.id)
session_binding = RuntimeToolBinding(tool_id=session_tool.id)
self.assertEqual(
brain._flow_tool(timeout_tool, "start", timeout_binding).timeout_secs,
7.0,
)
self.assertIsNone(
brain._flow_tool(session_tool, "start", session_binding).timeout_secs
)
async def test_flow_transition_honors_strict_agent_entry_mode(self):
brain = WorkflowBrain(
@@ -1961,9 +2002,12 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"id": "lookup_action",
"type": "action",
"data": {
"toolId": "lookup",
"resultAssignments": {
"order_status": "order.status"
"toolBinding": {
"toolId": "lookup",
"argumentBindings": {},
"resultAssignments": {
"order_status": "order.status"
},
},
},
},
@@ -1984,8 +2028,8 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def queue_frame(frame):
queued.append(frame)
async def execute(_tool, _arguments, *, result_assignments=None):
self.assertEqual(result_assignments, {"order_status": "order.status"})
async def execute(_tool, _arguments, *, binding=None, result_assignments=None):
self.assertEqual(binding.result_assignments, {"order_status": "order.status"})
brain._store.assign("order_status", "paid")
return {
"status": "ok",
@@ -2032,7 +2076,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(completed_event["workflowRevision"], brain._engine.revision)
self.assertNotIn("result", completed_event["outcome"])
async def test_action_result_assignment_modes_reach_tool_executor(self):
async def test_action_binding_reaches_tool_executor(self):
tool = RuntimeTool(
id="client_action",
name="客户端操作",
@@ -2051,25 +2095,30 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"id": "inherit_action",
"type": "action",
"data": {
"toolId": "client_action",
"resultAssignmentMode": "inherit",
"toolBinding": {
"toolId": "client_action",
"resultAssignments": {},
},
},
},
{
"id": "override_action",
"type": "action",
"data": {
"toolId": "client_action",
"resultAssignmentMode": "override",
"resultAssignments": {"choice": "action"},
"toolBinding": {
"toolId": "client_action",
"resultAssignments": {"choice": "action"},
},
},
},
{
"id": "none_action",
"type": "action",
"data": {
"toolId": "client_action",
"resultAssignmentMode": "none",
"toolBinding": {
"toolId": "client_action",
"resultAssignments": {},
},
},
},
],
@@ -2080,8 +2129,8 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
)
captured = []
async def execute(_tool, _arguments, *, result_assignments=None):
captured.append(result_assignments)
async def execute(_tool, _arguments, *, binding=None, result_assignments=None):
captured.append(binding.result_assignments)
return {"status": "ok", "updated_variables": []}
brain._runtime = BrainRuntime(
@@ -2098,7 +2147,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
await brain._enter_action("override_action")
await brain._enter_action("none_action")
self.assertEqual(captured, [None, {"choice": "action"}, {}])
self.assertEqual(captured, [{}, {"choice": "action"}, {}])
async def test_action_client_error_sets_error_status(self):
tool = RuntimeTool(
@@ -2118,7 +2167,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
{
"id": "action",
"type": "action",
"data": {"toolId": "client_action"},
"data": {"toolBinding": {"toolId": "client_action"}},
},
],
"edges": [],
@@ -2127,7 +2176,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
)
)
async def execute(_tool, _arguments, *, result_assignments=None):
async def execute(_tool, _arguments, *, binding=None, result_assignments=None):
return {
"status": "error",
"message": "用户关闭了确认弹窗",
@@ -2172,7 +2221,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
{
"id": "action",
"type": "action",
"data": {"toolId": "client_action"},
"data": {"toolBinding": {"toolId": "client_action"}},
},
{"id": "end", "type": "end", "data": {}},
],
@@ -2193,7 +2242,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
async def queue_frame(frame):
queued.append(frame)
async def execute(_tool, _arguments, *, result_assignments=None):
async def execute(_tool, _arguments, *, binding=None, result_assignments=None):
return {
"status": "error",
"message": "会话已结束",
@@ -2244,7 +2293,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"id": "block_action",
"type": "action",
"data": {
"toolId": "client_action",
"toolBinding": {"toolId": "client_action"},
"userInputPolicy": "block",
},
},
@@ -2252,7 +2301,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
"id": "queue_action",
"type": "action",
"data": {
"toolId": "client_action",
"toolBinding": {"toolId": "client_action"},
"userInputPolicy": "queue",
},
},
@@ -2264,7 +2313,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
)
input_states = []
async def execute(_tool, _arguments, *, result_assignments=None):
async def execute(_tool, _arguments, *, binding=None, result_assignments=None):
input_states.append("executing")
return {"status": "ok", "updated_variables": []}