feat: organize tools and enable qwen realtime calls

backend/services/brains/prompt_brain.py

@@ -9,7 +9,7 @@ from typing import Any
 from uuid import uuid4
 
 from loguru import logger
-from models import AssistantConfig
+from models import AssistantConfig, RuntimeToolBinding
 from pipecat.adapters.schemas.function_schema import FunctionSchema
 from pipecat.frames.frames import (
     LLMRunFrame,
@@ -28,8 +28,10 @@ from services.brains.base import (
     BaseBrain,
     BrainRuntime,
     BrainSpec,
+    RealtimeBrainRuntime,
     SessionVariableUpdate,
 )
+from services.brains.prompt_realtime_tools import PromptRealtimeToolController
 from services.action_runtime import (
     ActionOutcome,
     ActionInvocationCancelled,
@@ -46,7 +48,8 @@ from services.message_stage import (
     confirmation_context_message,
 )
 from services.runtime_variables import DynamicVariableError, DynamicVariableStore
-from services.system_tools import state_update_properties, system_tool_kind
+from services.system_tools import runtime_system_tools, state_update_properties, system_tool_kind
+from services.state_expressions import evaluate_update_rules
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 
@@ -70,6 +73,7 @@ class PromptBrain(BaseBrain):
         self._message_stages = MessageStageRunner()
         self._tool_by_id = {tool.id: tool for tool in cfg.tools}
         self._runtime: BrainRuntime | None = None
+        self._realtime_controller: PromptRealtimeToolController | None = None
         self._output: FixedSpeechOutput | None = None
         self._waiting_for_generated_end_speech = False
         self._preflight_finished = False
@@ -123,18 +127,33 @@ class PromptBrain(BaseBrain):
         self._startup_failed = False
         self._greeting_pending = False
         self._entry_dispatched = False
-        llm_tool_ids = (
-            set(cfg.llm_tool_ids) if cfg.llm_tool_ids is not None else None
-        )
         schemas: list[FunctionSchema] = []
         registered_names: set[str] = set()
-        for tool in cfg.tools:
-            if llm_tool_ids is not None and tool.id not in llm_tool_ids:
+        binding_by_id = {binding.tool_id: binding for binding in cfg.tool_bindings}
+        selected_tools = [
+            self._tool_by_id[binding.tool_id]
+            for binding in cfg.tool_bindings
+            if binding.tool_id in self._tool_by_id
+        ]
+        selected_tools.extend(runtime_system_tools(cfg.built_in_tools))
+        for tool in selected_tools:
+            if (
+                tool.type == "client"
+                and runtime.client_tools is not None
+                and getattr(runtime.client_tools, "supports", lambda _name: True)(
+                    tool.function_name
+                ) is False
+            ):
+                logger.warning(f"跳过客户端未上报的工具: {tool.function_name}")
                 continue
             if tool.type == "system":
                 schema, handler = self._make_system_tool(tool, runtime)
             elif tool.type in {"http", "mcp", "client"}:
-                schema, handler = self._make_remote_tool(tool, runtime)
+                schema, handler = self._make_remote_tool(
+                    tool,
+                    runtime,
+                    binding_by_id[tool.id],
+                )
             else:
                 continue
             if schema.name in registered_names:
@@ -152,6 +171,45 @@ class PromptBrain(BaseBrain):
             )
         runtime.set_tools(schemas)
 
+    async def setup_realtime(
+        self,
+        cfg: AssistantConfig,
+        runtime: RealtimeBrainRuntime,
+    ) -> None:
+        """Install Prompt-owned tools into a supported realtime provider."""
+
+        self._cfg = cfg
+        self._store = DynamicVariableStore.from_config(cfg)
+        self._tool_by_id = {tool.id: tool for tool in cfg.tools}
+        self._realtime_controller = PromptRealtimeToolController(
+            cfg=cfg,
+            store=self._store,
+            runtime=runtime,
+        )
+        runtime.realtime.set_speech_started_handler(
+            self.on_realtime_user_speech_started
+        )
+        set_response_done_handler = getattr(
+            runtime.realtime,
+            "set_response_done_handler",
+            None,
+        )
+        if callable(set_response_done_handler):
+            set_response_done_handler(self.on_realtime_response_done)
+        await self._realtime_controller.start()
+
+    async def on_realtime_user_speech_started(self) -> None:
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_user_speech_started()
+
+    async def on_realtime_response_done(
+        self,
+        content: str,
+        interrupted: bool,
+    ) -> None:
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_response_done(content, interrupted)
+
     async def run_preflight(self) -> None:
         if self._preflight_finished:
             return
@@ -367,20 +425,21 @@ class PromptBrain(BaseBrain):
         ]
 
     def _opening_actions_stage_spec(self) -> ActionStageSpec:
-        actions = tuple(
-            StageAction(
-                id=str(action.get("id") or "startup_action"),
-                tool=self._tool_by_id.get(
-                    str(action.get("tool_id") or action.get("toolId") or "")
-                ),
-                arguments=action.get("arguments") or {},
-                required=bool(action.get("required", True)),
-                invocation_id=self._actions.new_invocation_id(),
+        actions = []
+        for action in self._startup_actions("opening"):
+            raw_binding = action.get("tool_binding") or action.get("toolBinding") or {}
+            binding = RuntimeToolBinding(**raw_binding)
+            actions.append(
+                StageAction(
+                    id=str(action.get("id") or "startup_action"),
+                    tool=self._tool_by_id.get(binding.tool_id),
+                    binding=binding,
+                    required=bool(action.get("required", True)),
+                    invocation_id=self._actions.new_invocation_id(),
+                )
             )
-            for action in self._startup_actions("opening")
-        )
         return ActionStageSpec(
-            actions=actions,
+            actions=tuple(actions),
             input_policy="block",
         )
 
@@ -441,7 +500,9 @@ class PromptBrain(BaseBrain):
     async def _run_startup_actions(self, phase: str) -> bool:
         for action in self._startup_actions(phase):
             action_id = str(action.get("id") or "startup_action")
-            tool_id = str(action.get("tool_id") or action.get("toolId") or "")
+            raw_binding = action.get("tool_binding") or action.get("toolBinding") or {}
+            binding = RuntimeToolBinding(**raw_binding)
+            tool_id = binding.tool_id
             tool = self._tool_by_id.get(tool_id)
             invocation_id = self._actions.new_invocation_id()
             logger.info(
@@ -449,7 +510,7 @@ class PromptBrain(BaseBrain):
             )
             outcome = await self._actions.execute(
                 tool,
-                action.get("arguments") or {},
+                binding=binding,
                 invocation_id=invocation_id,
             )
             if outcome.updated_variables:
@@ -504,6 +565,8 @@ class PromptBrain(BaseBrain):
     async def on_assistant_text_start(self, _turn_id: str) -> None:
         if self._runtime is not None:
             self._runtime.call_end.begin_response()
+        if self._realtime_controller is not None:
+            self._realtime_controller.on_assistant_text_start()
 
     async def on_assistant_text_end(
         self,
@@ -520,14 +583,18 @@ class PromptBrain(BaseBrain):
                 await self._runtime.call_end.finish_after_current_speech(
                     has_text=bool(content.strip()) and not interrupted
                 )
+        if self._realtime_controller is not None:
+            await self._realtime_controller.on_assistant_text_end(
+                content,
+                interrupted,
+            )
 
     def _refresh_prompt(self) -> None:
         if self._dynamic_enabled and self._runtime is not None:
             self._runtime.set_system_prompt(self._store.render(self._cfg.prompt))
 
-    def _make_remote_tool(self, tool, runtime: BrainRuntime):
-        properties, required = self._tools.schema_parts(tool)
-        self._tools.register_secrets(tool)
+    def _make_remote_tool(self, tool, runtime: BrainRuntime, binding):
+        properties, required = self._tools.schema_parts(tool, binding)
         policy = policy_for_tool(tool)
 
         async def return_result(params: FunctionCallParams, result: dict) -> None:
@@ -551,7 +618,11 @@ class PromptBrain(BaseBrain):
                 toolType=tool.type,
             )
             try:
-                result = await self._tools.execute(tool, dict(params.arguments or {}))
+                result = await self._tools.execute(
+                    tool,
+                    dict(params.arguments or {}),
+                    binding=binding,
+                )
                 if result["updated_variables"]:
                     self._refresh_prompt()
                 await self._emit_trace(
@@ -690,7 +761,7 @@ class PromptBrain(BaseBrain):
         )
         return schema, end_call
 
-    # ---------- System 工具资源 ----------
+    # ---------- 平台内建对话工具 ----------
 
     def _make_system_tool(self, tool, runtime: BrainRuntime):
         kind = system_tool_kind(tool.definition or {})
@@ -707,7 +778,7 @@ class PromptBrain(BaseBrain):
         else:
             raise ValueError(f"未知系统工具: {kind}")
 
-        # System tools are platform-owned deterministic behavior. Keep their
+        # Built-ins are platform-owned deterministic behavior. Keep their
         # lifecycle and state semantics identical across production and tests;
         # only external HTTP, MCP, and client tools use an injected executor.
         return schema, handler
@@ -715,13 +786,22 @@ class PromptBrain(BaseBrain):
     def _make_update_state_tool(self, tool, runtime: BrainRuntime):
         """更新已声明的动态变量(会话状态),并让模型继续当前回答。"""
 
-        writable = state_update_properties(self._cfg.dynamic_variable_definitions)
+        config = (tool.definition or {}).get("config") or {}
+        rules = list(config.get("updates") or [])
+        writable = state_update_properties(
+            self._cfg.dynamic_variable_definitions,
+            rules=rules,
+        )
 
         async def update_state(params: FunctionCallParams) -> None:
-            state = dict(params.arguments or {})
             try:
+                state = evaluate_update_rules(
+                    rules,
+                    dict(self._store.values),
+                    dict(params.arguments or {}),
+                )
                 changed = self._store.assign_declared_many(state)
-            except DynamicVariableError as exc:
+            except (DynamicVariableError, ValueError) as exc:
                 await params.result_callback(
                     {"status": "error", "message": f"状态更新失败: {exc}"}
                 )