feat: organize tools and enable qwen realtime calls

2026-08-14 00:15:00 +08:00
parent eb111a3890
commit 5be21f5ccc
71 changed files with 5771 additions and 2026 deletions
@@ -8,7 +8,7 @@ from dataclasses import dataclass
 from typing import Any

 from loguru import logger
-from models import AssistantConfig, RuntimeTool
+from models import AssistantConfig, RuntimeTool, RuntimeToolBinding
 from pipecat.frames.frames import OutputTransportMessageUrgentFrame
 from pipecat.utils.time import time_now_iso8601

@@ -27,7 +27,8 @@ from services.message_stage import (
 from services.pipecat.call_lifecycle import playback_marker_for
 from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult
 from services.runtime_variables import DynamicVariableError, DynamicVariableStore
-from services.system_tools import state_update_properties, system_tool_kind
+from services.system_tools import runtime_system_tools, state_update_properties, system_tool_kind
+from services.state_expressions import evaluate_update_rules
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 from services.vision import (
@@ -434,27 +435,43 @@ class WorkflowRealtimeController:
            tools.append(tool)
            handlers[tool.name] = handler

-        for tool_id in stage.tool_ids:
-            runtime_tool = self._tool_by_id.get(str(tool_id))
+        for raw_binding in stage.tool_bindings:
+            binding = RuntimeToolBinding(**raw_binding)
+            runtime_tool = self._tool_by_id.get(binding.tool_id)
            if not runtime_tool:
                continue
-            if runtime_tool.type == "system":
-                built = self._system_tool(
-                    runtime_tool,
-                    node_id=node_id,
-                    transition_id=transition_id,
-                    state_variable_names=stage.state_variable_names,
+            if (
+                runtime_tool.type == "client"
+                and self._runtime.client_tools is not None
+                and getattr(
+                    self._runtime.client_tools,
+                    "supports",
+                    lambda _name: True,
+                )(runtime_tool.function_name) is False
+            ):
+                logger.warning(
+                    f"跳过客户端未上报的工具: {runtime_tool.function_name}"
                )
-            elif runtime_tool.type in {"http", "mcp", "client"}:
+                continue
+            if runtime_tool.type in {"http", "mcp", "client"}:
                built = self._business_tool(
                    runtime_tool,
                    node_id=node_id,
                    transition_id=transition_id,
+                    binding=binding,
                )
            else:
                built = None
            if built:
                add(*built)
+        for runtime_tool in runtime_system_tools(stage.built_in_tools):
+            built = self._system_tool(
+                runtime_tool,
+                node_id=node_id,
+                transition_id=transition_id,
+            )
+            if built:
+                add(*built)

        knowledge = self._knowledge_tool(node_id, transition_id)
        if knowledge:
@@ -478,7 +495,7 @@ class WorkflowRealtimeController:
                if not self._is_current(node_id, transition_id):
                    return RealtimeToolResult(
                        {"status": "stale", "message": "当前 Agent 已经切换。"},
-                        continue_response=False,
+                        follow_up="none",
                    )
                activation = await self._follow_edge(
                    edge,
@@ -489,7 +506,7 @@ class WorkflowRealtimeController:
                    "status": "success",
                    "targetNodeId": str(edge.get("target") or ""),
                },
-                continue_response=activation.continue_response,
+                follow_up=("continue" if activation.continue_response else "none"),
                after_output=activation.after_output,
            )

@@ -507,14 +524,18 @@ class WorkflowRealtimeController:
        *,
        node_id: str,
        transition_id: int,
+        binding: RuntimeToolBinding,
    ) -> tuple[RealtimeTool, ToolHandler]:
-        properties, required = self._tools.schema_parts(tool)
-        self._tools.register_secrets(tool)
+        properties, required = self._tools.schema_parts(tool, binding)
        policy = policy_for_tool(tool)

        async def handler(arguments: dict[str, Any]) -> RealtimeToolResult:
            try:
-                result = await self._tools.execute(tool, arguments)
+                result = await self._tools.execute(
+                    tool,
+                    arguments,
+                    binding=binding,
+                )
            except ToolExecutionError as exc:
                return RealtimeToolResult(
                    {"status": "error", "message": str(exc)}
@@ -527,7 +548,7 @@ class WorkflowRealtimeController:
                            "status": "stale",
                            "message": "工具完成时当前 Agent 已经切换。",
                        },
-                        continue_response=False,
+                        follow_up="none",
                    )
                activation = await self._after_variables_changed(
                    list(result.get("updated_variables") or []),
@@ -535,10 +556,14 @@ class WorkflowRealtimeController:
                )
            return RealtimeToolResult(
                result,
-                continue_response=(
-                    activation.continue_response
-                    if activation
-                    else policy.runs_llm_after_result
+                follow_up=(
+                    "continue"
+                    if (
+                        activation.continue_response
+                        if activation
+                        else policy.runs_llm_after_result
+                    )
+                    else "none"
                ),
                after_output=activation.after_output if activation else None,
            )
@@ -575,7 +600,7 @@ class WorkflowRealtimeController:
            if not self._is_current(node_id, transition_id):
                return RealtimeToolResult(
                    {"status": "stale", "message": "当前 Agent 已经切换。"},
-                    continue_response=False,
+                    follow_up="none",
                )
            try:
                question = resolve_vision_question(
@@ -609,7 +634,7 @@ class WorkflowRealtimeController:
            if not self._is_current(node_id, transition_id):
                return RealtimeToolResult(
                    {"status": "stale", "message": "获取画面时 Agent 已经切换。"},
-                    continue_response=False,
+                    follow_up="none",
                )

            try:
@@ -632,7 +657,7 @@ class WorkflowRealtimeController:
            if not self._is_current(node_id, transition_id):
                return RealtimeToolResult(
                    {"status": "stale", "message": "分析画面时 Agent 已经切换。"},
-                    continue_response=False,
+                    follow_up="none",
                )
            return RealtimeToolResult(
                {
@@ -662,25 +687,21 @@ class WorkflowRealtimeController:
        *,
        node_id: str,
        transition_id: int,
-        state_variable_names: tuple[str, ...],
    ) -> tuple[RealtimeTool, ToolHandler] | None:
        kind = system_tool_kind(tool.definition or {})
        if kind == "update_state":
-            allowed = frozenset(state_variable_names)
+            config = (tool.definition or {}).get("config") or {}
+            rules = list(config.get("updates") or [])

            async def update_state(arguments: dict[str, Any]) -> RealtimeToolResult:
-                unauthorized = sorted(set(arguments) - allowed)
-                if unauthorized:
-                    return RealtimeToolResult(
-                        {
-                            "status": "error",
-                            "message": "状态变量未获当前节点授权:"
-                            + ",".join(unauthorized),
-                        }
-                    )
                try:
-                    changed = self._store.assign_declared_many(arguments)
-                except DynamicVariableError as exc:
+                    values = evaluate_update_rules(
+                        rules,
+                        dict(self._store.values),
+                        arguments,
+                    )
+                    changed = self._store.assign_declared_many(values)
+                except (DynamicVariableError, ValueError) as exc:
                    return RealtimeToolResult(
                        {"status": "error", "message": f"状态更新失败:{exc}"}
                    )
@@ -688,7 +709,7 @@ class WorkflowRealtimeController:
                    if not self._is_current(node_id, transition_id):
                        return RealtimeToolResult(
                            {"status": "stale", "message": "当前 Agent 已经切换。"},
-                            continue_response=False,
+                            follow_up="none",
                        )
                    activation = await self._after_variables_changed(
                        changed,
@@ -700,8 +721,10 @@ class WorkflowRealtimeController:
                        "changed": changed,
                        "variables": self._store.public_values(),
                    },
-                    continue_response=(
-                        activation.continue_response if activation else True
+                    follow_up=(
+                        "continue"
+                        if (activation.continue_response if activation else True)
+                        else "none"
                    ),
                    after_output=activation.after_output if activation else None,
                )
@@ -712,7 +735,7 @@ class WorkflowRealtimeController:
                    description=tool.description or "更新已授权的动态变量。",
                    properties=state_update_properties(
                        self._cfg.dynamic_variable_definitions,
-                        allowed_names=state_variable_names,
+                        rules=rules,
                    ),
                ),
                update_state,
@@ -725,7 +748,7 @@ class WorkflowRealtimeController:
                        "action": "skip_turn",
                        "reason": str(arguments.get("reason") or ""),
                    },
-                    continue_response=False,
+                    follow_up="suppress",
                )

            return (
@@ -813,7 +836,7 @@ class WorkflowRealtimeController:
            if not self._is_current(node_id, transition_id):
                return RealtimeToolResult(
                    {"status": "stale", "message": "当前 Agent 已经切换。"},
-                    continue_response=False,
+                    follow_up="none",
                )
            query = str(arguments.get("query") or "").strip()
            if not query:
@@ -932,15 +955,15 @@ class WorkflowRealtimeController:
        if self._runtime.set_input_enabled:
            self._runtime.set_input_enabled(False)
        data = self._engine.data(node_id)
-        tool = self._tool_by_id.get(str(data.get("toolId") or ""))
+        binding = RuntimeToolBinding(**(data.get("toolBinding") or {}))
+        tool = self._tool_by_id.get(binding.tool_id)
        result = await self._action_stages.run(
            ActionStageSpec(
                actions=(
                    StageAction(
                        id=node_id,
                        tool=tool,
-                        arguments=data.get("arguments") or {},
-                        result_assignments=self._action_result_assignments(data),
+                        binding=binding,
                    ),
                ),
                input_policy="block",
@@ -961,12 +984,9 @@ class WorkflowRealtimeController:
    async def _enter_update_state(self, node_id: str) -> None:
        self._state.enter(node_id, WorkflowStatus.RUNNING_ACTION)
        await self._emit_active(node_id)
-        rendered = self._store.render_data(
-            self._engine.data(node_id).get("assignments") or {}
-        )
-        if not isinstance(rendered, dict):
-            raise DynamicVariableError("Update State 节点赋值必须是对象")
-        changed = self._store.assign_declared_many(rendered)
+        updates = self._engine.data(node_id).get("updates") or []
+        values = evaluate_update_rules(updates, dict(self._store.values))
+        changed = self._store.assign_declared_many(values)
        if changed:
            await self._emit_variables("update_state", node_id, changed)

@@ -1058,18 +1078,6 @@ class WorkflowRealtimeController:
            self._engine.data(node_id).get("completionPolicy") or "playback"
        )

-    @staticmethod
-    def _action_result_assignments(
-        data: dict[str, Any],
-    ) -> dict[str, str] | None:
-        mode = str(data.get("resultAssignmentMode") or "none")
-        if mode == "inherit":
-            return None
-        if mode == "override":
-            value = data.get("resultAssignments")
-            return dict(value) if isinstance(value, dict) else {}
-        return {}
-
    def _set_last_action(self, outcome) -> None:
        self._store.values.update(
            {