"""Prompt-owned tools for Qwen speech-to-speech sessions.""" from __future__ import annotations from collections.abc import Awaitable, Callable from time import monotonic from typing import Any from uuid import uuid4 from loguru import logger from models import AssistantConfig, RuntimeTool, RuntimeToolBinding from pipecat.frames.frames import OutputTransportMessageUrgentFrame from pipecat.utils.time import time_now_iso8601 from services.brains.base import RealtimeBrainRuntime from services.pipecat.call_lifecycle import playback_marker_for from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult from services.runtime_variables import DynamicVariableError, DynamicVariableStore from services.state_expressions import evaluate_update_rules from services.system_tools import ( RESERVED_TOOL_NAMES, runtime_system_tools, state_update_properties, system_tool_kind, ) from services.tool_executor import ToolExecutor from services.tool_policy import policy_for_tool ToolHandler = Callable[[dict[str, Any]], Awaitable[RealtimeToolResult]] SUPPORTED_BUILT_INS = frozenset( {"end_conversation", "update_state", "request_human_handoff"} ) MAX_PUBLIC_TOOL_ERROR = 512 class PromptRealtimeToolController: """Install and execute Prompt tools without modifying Pipecat itself.""" def __init__( self, *, cfg: AssistantConfig, store: DynamicVariableStore, runtime: RealtimeBrainRuntime, ) -> None: self._cfg = cfg self._store = store self._runtime = runtime self._executor = ToolExecutor(store, client_tools=runtime.client_tools) self._tools_by_id = {tool.id: tool for tool in cfg.tools} self._handlers: dict[str, ToolHandler] = {} self._input_locks = 0 self._handoff_requested = False self._waiting_generated_end = False async def start(self) -> None: if self._cfg.realtime_interface_type != "qwen-audio-realtime": if self._cfg.tool_bindings or runtime_system_tools( self._cfg.built_in_tools ): raise ValueError( "当前 Realtime Provider 不支持 Prompt 工具调用" ) return tools = await self._build_tools() self._runtime.realtime.set_tool_dispatcher(self.dispatch_tool) await self._runtime.realtime.update_session( self._store.render(self._cfg.prompt), tools, ) async def dispatch_tool( self, name: str, arguments: dict[str, Any], _call_id: str, ) -> RealtimeToolResult: handler = self._handlers.get(name) if handler is None: return RealtimeToolResult( {"status": "error", "message": f"当前会话没有工具:{name}"} ) return await handler(arguments) async def on_user_speech_started(self) -> None: """Cancel only an explicitly interruptible pending ending.""" if self._runtime.call_end.cancel_pending_end(): self._waiting_generated_end = False self._set_input_enabled(True, force=True) def on_assistant_text_start(self) -> None: """Reset transport speech tracking for the response that just started.""" self._runtime.call_end.begin_response() async def on_assistant_text_end( self, content: str, interrupted: bool, ) -> None: if not self._waiting_generated_end: return self._waiting_generated_end = False if self._runtime.call_end.ending: await self._runtime.call_end.finish_after_current_speech( has_text=bool(content.strip()) and not interrupted ) async def on_response_done(self, content: str, interrupted: bool) -> None: """Finish a generated ending even when the response has no transcript.""" if self._waiting_generated_end: await self.on_assistant_text_end(content, interrupted) async def _build_tools(self) -> list[RealtimeTool]: result: list[RealtimeTool] = [] binding_by_id = { binding.tool_id: binding for binding in self._cfg.tool_bindings } async def add( tool: RealtimeTool, handler: ToolHandler, *, source_id: str, ) -> None: if tool.name in self._handlers: await self._emit_trace( "tool_skipped", toolId=source_id, functionName=tool.name, reason="duplicate_function_name", ) return self._handlers[tool.name] = handler result.append(tool) for binding in self._cfg.tool_bindings: runtime_tool = self._tools_by_id.get(binding.tool_id) if runtime_tool is None or runtime_tool.type not in { "http", "mcp", "client", }: continue if runtime_tool.function_name in RESERVED_TOOL_NAMES: raise ValueError( f"外部工具使用了平台保留函数名:{runtime_tool.function_name}" ) if ( runtime_tool.type == "client" and ( self._runtime.client_tools is None or getattr( self._runtime.client_tools, "supports", lambda _name: True, )(runtime_tool.function_name) is False ) ): logger.warning( f"跳过客户端未上报的 Realtime 工具: " f"{runtime_tool.function_name}" ) await self._emit_trace( "tool_skipped", toolId=runtime_tool.id, functionName=runtime_tool.function_name, toolType=runtime_tool.type, reason="client_capability_missing", ) continue built = self._external_tool(runtime_tool, binding_by_id[binding.tool_id]) await add(*built, source_id=runtime_tool.id) for runtime_tool in runtime_system_tools(self._cfg.built_in_tools): kind = system_tool_kind(runtime_tool.definition or {}) if kind == "skip_turn": raise ValueError("Prompt Realtime 暂不支持 skip_turn") if kind not in SUPPORTED_BUILT_INS: continue built = self._system_tool(runtime_tool) if built is not None: await add(*built, source_id=runtime_tool.id) return result def _external_tool( self, tool: RuntimeTool, binding: RuntimeToolBinding, ) -> tuple[RealtimeTool, ToolHandler]: properties, required = self._executor.schema_parts(tool, binding) policy = policy_for_tool(tool) async def handler(arguments: dict[str, Any]) -> RealtimeToolResult: invocation_id = f"tool_{uuid4().hex[:20]}" started_at = monotonic() await self._emit_trace( "tool_started", invocationId=invocation_id, toolId=tool.id, toolName=tool.name, functionName=tool.function_name, toolType=tool.type, ) if not policy.allow_interruptions: self._lock_input() try: result = await self._executor.execute( tool, arguments, binding=binding, ) changed = list(result.get("updated_variables") or []) if changed: await self._publish_variables("tool", changed) await self._refresh_instructions() await self._emit_trace( "tool_completed" if result.get("status") == "ok" else "tool_failed", invocationId=invocation_id, toolId=tool.id, toolName=tool.name, functionName=tool.function_name, toolType=tool.type, status=str(result.get("status") or "unknown"), durationMs=max(0, round((monotonic() - started_at) * 1000)), updatedVariables=changed, resultKeys=sorted(str(name) for name in result if name != "data"), ) follow_up = ( "continue" if policy.runs_llm_after_result else "none" ) return RealtimeToolResult( result, follow_up=follow_up, ) except Exception as exc: # noqa: BLE001 - sanitize provider errors message = self._public_tool_error(exc) await self._emit_trace( "tool_failed", invocationId=invocation_id, toolId=tool.id, toolName=tool.name, functionName=tool.function_name, toolType=tool.type, status="error", durationMs=max(0, round((monotonic() - started_at) * 1000)), error=message, ) follow_up = ( "continue" if policy.runs_llm_after_result else "none" ) return RealtimeToolResult( {"status": "error", "message": message}, follow_up=follow_up, ) finally: if not policy.allow_interruptions: self._unlock_input() return ( RealtimeTool( name=tool.function_name, description=tool.description or f"调用 {tool.name}", properties=properties, required=tuple(required), ), handler, ) def _system_tool( self, tool: RuntimeTool, ) -> tuple[RealtimeTool, ToolHandler] | None: kind = system_tool_kind(tool.definition or {}) config = (tool.definition or {}).get("config") or {} if kind == "update_state": rules = list(config.get("updates") or []) async def update_state(arguments: dict[str, Any]) -> RealtimeToolResult: try: values = evaluate_update_rules( rules, dict(self._store.values), arguments, ) changed = self._store.assign_declared_many(values) except (DynamicVariableError, ValueError) as exc: return RealtimeToolResult( { "status": "error", "message": f"状态更新失败:{self._public_error(exc)}", } ) if changed: await self._publish_variables("update_state", changed) await self._refresh_instructions() return RealtimeToolResult( { "status": "success", "changed": changed, "variables": self._store.public_values(), } ) return ( RealtimeTool( name=tool.function_name, description=tool.description or "更新已授权的动态变量。", properties=state_update_properties( self._cfg.dynamic_variable_definitions, rules=rules, ), ), update_state, ) if kind == "request_human_handoff": async def request_handoff( arguments: dict[str, Any], ) -> RealtimeToolResult: reason = str(arguments.get("reason") or "human_handoff") if not self._handoff_requested: self._handoff_requested = True self._store.values["system__handoff_status"] = "requested" await self._runtime.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "handoff-requested", "source": "prompt-system-tool", "reason": reason, "message": "用户请求转接人工服务。", } ) ) await self._publish_variables( "human_handoff", ["system__handoff_status"], ) await self._refresh_instructions() return RealtimeToolResult( { "status": "requested", "action": "human_handoff_requested", "message": "人工接管请求已提交，请告知用户正在等待人工响应。", } ) return ( RealtimeTool( name=tool.function_name, description=tool.description or "提交人工接管请求。", properties={ "reason": {"type": "string", "description": "转接原因"} }, ), request_handoff, ) if kind == "end_conversation": capture_reason = bool(config.get("capture_reason", True)) allow_end_interruption = bool( config.get("allow_end_interruption", False) ) message_type = str(config.get("message_type") or "none") custom_message = str(config.get("custom_message") or "").strip() async def end_conversation( arguments: dict[str, Any], ) -> RealtimeToolResult: if self._runtime.call_end.ending: return RealtimeToolResult( {"status": "success", "action": "ending_call"}, follow_up="suppress", ) reason = str( arguments.get("reason") or "end_conversation" ).strip() self._runtime.call_end.begin( reason, interruptible=allow_end_interruption, ) if not allow_end_interruption: self._set_input_enabled(False, force=True) uses_custom_message = ( message_type == "custom" and bool(custom_message) ) if uses_custom_message: return RealtimeToolResult( {"status": "success", "action": "ending_call"}, follow_up="suppress", after_output=lambda: self._play_end_message(custom_message), ) self._waiting_generated_end = True return RealtimeToolResult( { "status": "success", "action": "ending_call", "message": "请用一句简短、礼貌的结束语结束对话。", } ) properties = ( { "reason": { "type": "string", "description": "结束本次通话的简短原因。", } } if capture_reason else {} ) return ( RealtimeTool( name=tool.function_name, description=tool.description or "礼貌结束本次对话。", properties=properties, required=("reason",) if capture_reason else (), ), end_conversation, ) return None async def _play_end_message(self, content: str) -> None: if not self._runtime.call_end.ending: return turn_id = uuid4().hex timestamp = time_now_iso8601() for message in ( {"type": "assistant-text-start", "turn_id": turn_id, "timestamp": timestamp}, {"type": "assistant-text-delta", "turn_id": turn_id, "delta": content}, { "type": "assistant-text-end", "turn_id": turn_id, "content": content, "interrupted": False, }, ): await self._runtime.queue_frame( OutputTransportMessageUrgentFrame(message=message) ) track_speech = getattr(self._runtime.call_end, "track_speech", None) playback_completion = track_speech() if callable(track_speech) else None provider_completion = await self._runtime.realtime.speak_fixed( content, suppress_transcript=True, ) marker = playback_marker_for(playback_completion) if marker is not None: if provider_completion is not None: await provider_completion await self._runtime.queue_frame(marker) marker.completion.mark_queued() await self._runtime.call_end.arm_after_tracked_speech() await playback_completion return self._runtime.call_end.arm_after_speech() async def _refresh_instructions(self) -> None: await self._runtime.realtime.update_instructions( self._store.render(self._cfg.prompt) ) async def _publish_variables(self, reason: str, changed: list[str]) -> None: await self._runtime.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "session-variables", "reason": reason, "variables": self._store.public_values(), "changed": changed, } ) ) async def _emit_trace(self, event: str, **details: Any) -> None: try: await self._runtime.queue_frame( OutputTransportMessageUrgentFrame( message={ "type": "workflow-event", "eventId": f"wfe_{uuid4().hex[:20]}", "event": event, "timestamp": time_now_iso8601(), "sessionId": self._runtime.session_id, "source": "prompt-realtime", **details, } ) ) except Exception as exc: # noqa: BLE001 - trace must not break tools logger.warning(f"发送 Prompt Realtime 工具轨迹失败:{exc}") def _lock_input(self) -> None: self._input_locks += 1 self._set_input_enabled(False) def _unlock_input(self) -> None: self._input_locks = max(0, self._input_locks - 1) if self._input_locks == 0 and not self._runtime.call_end.ending: self._set_input_enabled(True) def _set_input_enabled(self, enabled: bool, *, force: bool = False) -> None: if self._runtime.set_input_enabled is None: return if force or enabled is False or self._input_locks == 0: self._runtime.set_input_enabled(enabled) @staticmethod def _public_error(exc: Exception) -> str: message = str(exc).strip() or type(exc).__name__ return message[:MAX_PUBLIC_TOOL_ERROR] @staticmethod def _public_tool_error(exc: Exception) -> str: """Return actionable errors without exposing URLs, payloads, or secrets.""" message = str(exc).strip() if "超时" in message: return "工具调用超时" if message.startswith("客户端未注册工具"): return message[:MAX_PUBLIC_TOOL_ERROR] if any(token in message for token in ("参数", "变量", "绑定")): return message[:MAX_PUBLIC_TOOL_ERROR] return "工具调用失败，请稍后重试"