531 lines
20 KiB
Python
531 lines
20 KiB
Python
|
"""Prompt-owned tools for Qwen speech-to-speech sessions."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
from collections.abc import Awaitable, Callable
|
||
|
|
from time import monotonic
|
||
|
|
from typing import Any
|
||
|
|
from uuid import uuid4
|
||
|
|
|
||
|
|
from loguru import logger
|
||
|
|
from models import AssistantConfig, RuntimeTool, RuntimeToolBinding
|
||
|
|
from pipecat.frames.frames import OutputTransportMessageUrgentFrame
|
||
|
|
from pipecat.utils.time import time_now_iso8601
|
||
|
|
|
||
|
|
from services.brains.base import RealtimeBrainRuntime
|
||
|
|
from services.pipecat.call_lifecycle import playback_marker_for
|
||
|
|
from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult
|
||
|
|
from services.runtime_variables import DynamicVariableError, DynamicVariableStore
|
||
|
|
from services.state_expressions import evaluate_update_rules
|
||
|
|
from services.system_tools import (
|
||
|
|
RESERVED_TOOL_NAMES,
|
||
|
|
runtime_system_tools,
|
||
|
|
state_update_properties,
|
||
|
|
system_tool_kind,
|
||
|
|
)
|
||
|
|
from services.tool_executor import ToolExecutor
|
||
|
|
from services.tool_policy import policy_for_tool
|
||
|
|
|
||
|
|
|
||
|
|
ToolHandler = Callable[[dict[str, Any]], Awaitable[RealtimeToolResult]]
|
||
|
|
SUPPORTED_BUILT_INS = frozenset(
|
||
|
|
{"end_conversation", "update_state", "request_human_handoff"}
|
||
|
|
)
|
||
|
|
MAX_PUBLIC_TOOL_ERROR = 512
|
||
|
|
|
||
|
|
|
||
|
|
class PromptRealtimeToolController:
|
||
|
|
"""Install and execute Prompt tools without modifying Pipecat itself."""
|
||
|
|
|
||
|
|
def __init__(
|
||
|
|
self,
|
||
|
|
*,
|
||
|
|
cfg: AssistantConfig,
|
||
|
|
store: DynamicVariableStore,
|
||
|
|
runtime: RealtimeBrainRuntime,
|
||
|
|
) -> None:
|
||
|
|
self._cfg = cfg
|
||
|
|
self._store = store
|
||
|
|
self._runtime = runtime
|
||
|
|
self._executor = ToolExecutor(store, client_tools=runtime.client_tools)
|
||
|
|
self._tools_by_id = {tool.id: tool for tool in cfg.tools}
|
||
|
|
self._handlers: dict[str, ToolHandler] = {}
|
||
|
|
self._input_locks = 0
|
||
|
|
self._handoff_requested = False
|
||
|
|
self._waiting_generated_end = False
|
||
|
|
|
||
|
|
async def start(self) -> None:
|
||
|
|
if self._cfg.realtime_interface_type != "qwen-audio-realtime":
|
||
|
|
if self._cfg.tool_bindings or runtime_system_tools(
|
||
|
|
self._cfg.built_in_tools
|
||
|
|
):
|
||
|
|
raise ValueError(
|
||
|
|
"当前 Realtime Provider 不支持 Prompt 工具调用"
|
||
|
|
)
|
||
|
|
return
|
||
|
|
tools = await self._build_tools()
|
||
|
|
self._runtime.realtime.set_tool_dispatcher(self.dispatch_tool)
|
||
|
|
await self._runtime.realtime.update_session(
|
||
|
|
self._store.render(self._cfg.prompt),
|
||
|
|
tools,
|
||
|
|
)
|
||
|
|
|
||
|
|
async def dispatch_tool(
|
||
|
|
self,
|
||
|
|
name: str,
|
||
|
|
arguments: dict[str, Any],
|
||
|
|
_call_id: str,
|
||
|
|
) -> RealtimeToolResult:
|
||
|
|
handler = self._handlers.get(name)
|
||
|
|
if handler is None:
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{"status": "error", "message": f"当前会话没有工具:{name}"}
|
||
|
|
)
|
||
|
|
return await handler(arguments)
|
||
|
|
|
||
|
|
async def on_user_speech_started(self) -> None:
|
||
|
|
"""Cancel only an explicitly interruptible pending ending."""
|
||
|
|
|
||
|
|
if self._runtime.call_end.cancel_pending_end():
|
||
|
|
self._waiting_generated_end = False
|
||
|
|
self._set_input_enabled(True, force=True)
|
||
|
|
|
||
|
|
def on_assistant_text_start(self) -> None:
|
||
|
|
"""Reset transport speech tracking for the response that just started."""
|
||
|
|
|
||
|
|
self._runtime.call_end.begin_response()
|
||
|
|
|
||
|
|
async def on_assistant_text_end(
|
||
|
|
self,
|
||
|
|
content: str,
|
||
|
|
interrupted: bool,
|
||
|
|
) -> None:
|
||
|
|
if not self._waiting_generated_end:
|
||
|
|
return
|
||
|
|
self._waiting_generated_end = False
|
||
|
|
if self._runtime.call_end.ending:
|
||
|
|
await self._runtime.call_end.finish_after_current_speech(
|
||
|
|
has_text=bool(content.strip()) and not interrupted
|
||
|
|
)
|
||
|
|
|
||
|
|
async def on_response_done(self, content: str, interrupted: bool) -> None:
|
||
|
|
"""Finish a generated ending even when the response has no transcript."""
|
||
|
|
|
||
|
|
if self._waiting_generated_end:
|
||
|
|
await self.on_assistant_text_end(content, interrupted)
|
||
|
|
|
||
|
|
async def _build_tools(self) -> list[RealtimeTool]:
|
||
|
|
result: list[RealtimeTool] = []
|
||
|
|
binding_by_id = {
|
||
|
|
binding.tool_id: binding for binding in self._cfg.tool_bindings
|
||
|
|
}
|
||
|
|
|
||
|
|
async def add(
|
||
|
|
tool: RealtimeTool,
|
||
|
|
handler: ToolHandler,
|
||
|
|
*,
|
||
|
|
source_id: str,
|
||
|
|
) -> None:
|
||
|
|
if tool.name in self._handlers:
|
||
|
|
await self._emit_trace(
|
||
|
|
"tool_skipped",
|
||
|
|
toolId=source_id,
|
||
|
|
functionName=tool.name,
|
||
|
|
reason="duplicate_function_name",
|
||
|
|
)
|
||
|
|
return
|
||
|
|
self._handlers[tool.name] = handler
|
||
|
|
result.append(tool)
|
||
|
|
|
||
|
|
for binding in self._cfg.tool_bindings:
|
||
|
|
runtime_tool = self._tools_by_id.get(binding.tool_id)
|
||
|
|
if runtime_tool is None or runtime_tool.type not in {
|
||
|
|
"http",
|
||
|
|
"mcp",
|
||
|
|
"client",
|
||
|
|
}:
|
||
|
|
continue
|
||
|
|
if runtime_tool.function_name in RESERVED_TOOL_NAMES:
|
||
|
|
raise ValueError(
|
||
|
|
f"外部工具使用了平台保留函数名:{runtime_tool.function_name}"
|
||
|
|
)
|
||
|
|
if (
|
||
|
|
runtime_tool.type == "client"
|
||
|
|
and (
|
||
|
|
self._runtime.client_tools is None
|
||
|
|
or getattr(
|
||
|
|
self._runtime.client_tools,
|
||
|
|
"supports",
|
||
|
|
lambda _name: True,
|
||
|
|
)(runtime_tool.function_name)
|
||
|
|
is False
|
||
|
|
)
|
||
|
|
):
|
||
|
|
logger.warning(
|
||
|
|
f"跳过客户端未上报的 Realtime 工具: "
|
||
|
|
f"{runtime_tool.function_name}"
|
||
|
|
)
|
||
|
|
await self._emit_trace(
|
||
|
|
"tool_skipped",
|
||
|
|
toolId=runtime_tool.id,
|
||
|
|
functionName=runtime_tool.function_name,
|
||
|
|
toolType=runtime_tool.type,
|
||
|
|
reason="client_capability_missing",
|
||
|
|
)
|
||
|
|
continue
|
||
|
|
built = self._external_tool(runtime_tool, binding_by_id[binding.tool_id])
|
||
|
|
await add(*built, source_id=runtime_tool.id)
|
||
|
|
|
||
|
|
for runtime_tool in runtime_system_tools(self._cfg.built_in_tools):
|
||
|
|
kind = system_tool_kind(runtime_tool.definition or {})
|
||
|
|
if kind == "skip_turn":
|
||
|
|
raise ValueError("Prompt Realtime 暂不支持 skip_turn")
|
||
|
|
if kind not in SUPPORTED_BUILT_INS:
|
||
|
|
continue
|
||
|
|
built = self._system_tool(runtime_tool)
|
||
|
|
if built is not None:
|
||
|
|
await add(*built, source_id=runtime_tool.id)
|
||
|
|
return result
|
||
|
|
|
||
|
|
def _external_tool(
|
||
|
|
self,
|
||
|
|
tool: RuntimeTool,
|
||
|
|
binding: RuntimeToolBinding,
|
||
|
|
) -> tuple[RealtimeTool, ToolHandler]:
|
||
|
|
properties, required = self._executor.schema_parts(tool, binding)
|
||
|
|
policy = policy_for_tool(tool)
|
||
|
|
|
||
|
|
async def handler(arguments: dict[str, Any]) -> RealtimeToolResult:
|
||
|
|
invocation_id = f"tool_{uuid4().hex[:20]}"
|
||
|
|
started_at = monotonic()
|
||
|
|
await self._emit_trace(
|
||
|
|
"tool_started",
|
||
|
|
invocationId=invocation_id,
|
||
|
|
toolId=tool.id,
|
||
|
|
toolName=tool.name,
|
||
|
|
functionName=tool.function_name,
|
||
|
|
toolType=tool.type,
|
||
|
|
)
|
||
|
|
if not policy.allow_interruptions:
|
||
|
|
self._lock_input()
|
||
|
|
try:
|
||
|
|
result = await self._executor.execute(
|
||
|
|
tool,
|
||
|
|
arguments,
|
||
|
|
binding=binding,
|
||
|
|
)
|
||
|
|
changed = list(result.get("updated_variables") or [])
|
||
|
|
if changed:
|
||
|
|
await self._publish_variables("tool", changed)
|
||
|
|
await self._refresh_instructions()
|
||
|
|
await self._emit_trace(
|
||
|
|
"tool_completed"
|
||
|
|
if result.get("status") == "ok"
|
||
|
|
else "tool_failed",
|
||
|
|
invocationId=invocation_id,
|
||
|
|
toolId=tool.id,
|
||
|
|
toolName=tool.name,
|
||
|
|
functionName=tool.function_name,
|
||
|
|
toolType=tool.type,
|
||
|
|
status=str(result.get("status") or "unknown"),
|
||
|
|
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||
|
|
updatedVariables=changed,
|
||
|
|
resultKeys=sorted(str(name) for name in result if name != "data"),
|
||
|
|
)
|
||
|
|
follow_up = (
|
||
|
|
"continue" if policy.runs_llm_after_result else "none"
|
||
|
|
)
|
||
|
|
return RealtimeToolResult(
|
||
|
|
result,
|
||
|
|
follow_up=follow_up,
|
||
|
|
)
|
||
|
|
except Exception as exc: # noqa: BLE001 - sanitize provider errors
|
||
|
|
message = self._public_tool_error(exc)
|
||
|
|
await self._emit_trace(
|
||
|
|
"tool_failed",
|
||
|
|
invocationId=invocation_id,
|
||
|
|
toolId=tool.id,
|
||
|
|
toolName=tool.name,
|
||
|
|
functionName=tool.function_name,
|
||
|
|
toolType=tool.type,
|
||
|
|
status="error",
|
||
|
|
durationMs=max(0, round((monotonic() - started_at) * 1000)),
|
||
|
|
error=message,
|
||
|
|
)
|
||
|
|
follow_up = (
|
||
|
|
"continue" if policy.runs_llm_after_result else "none"
|
||
|
|
)
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{"status": "error", "message": message},
|
||
|
|
follow_up=follow_up,
|
||
|
|
)
|
||
|
|
finally:
|
||
|
|
if not policy.allow_interruptions:
|
||
|
|
self._unlock_input()
|
||
|
|
|
||
|
|
return (
|
||
|
|
RealtimeTool(
|
||
|
|
name=tool.function_name,
|
||
|
|
description=tool.description or f"调用 {tool.name}",
|
||
|
|
properties=properties,
|
||
|
|
required=tuple(required),
|
||
|
|
),
|
||
|
|
handler,
|
||
|
|
)
|
||
|
|
|
||
|
|
def _system_tool(
|
||
|
|
self,
|
||
|
|
tool: RuntimeTool,
|
||
|
|
) -> tuple[RealtimeTool, ToolHandler] | None:
|
||
|
|
kind = system_tool_kind(tool.definition or {})
|
||
|
|
config = (tool.definition or {}).get("config") or {}
|
||
|
|
if kind == "update_state":
|
||
|
|
rules = list(config.get("updates") or [])
|
||
|
|
|
||
|
|
async def update_state(arguments: dict[str, Any]) -> RealtimeToolResult:
|
||
|
|
try:
|
||
|
|
values = evaluate_update_rules(
|
||
|
|
rules,
|
||
|
|
dict(self._store.values),
|
||
|
|
arguments,
|
||
|
|
)
|
||
|
|
changed = self._store.assign_declared_many(values)
|
||
|
|
except (DynamicVariableError, ValueError) as exc:
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{
|
||
|
|
"status": "error",
|
||
|
|
"message": f"状态更新失败:{self._public_error(exc)}",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
if changed:
|
||
|
|
await self._publish_variables("update_state", changed)
|
||
|
|
await self._refresh_instructions()
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{
|
||
|
|
"status": "success",
|
||
|
|
"changed": changed,
|
||
|
|
"variables": self._store.public_values(),
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
return (
|
||
|
|
RealtimeTool(
|
||
|
|
name=tool.function_name,
|
||
|
|
description=tool.description or "更新已授权的动态变量。",
|
||
|
|
properties=state_update_properties(
|
||
|
|
self._cfg.dynamic_variable_definitions,
|
||
|
|
rules=rules,
|
||
|
|
),
|
||
|
|
),
|
||
|
|
update_state,
|
||
|
|
)
|
||
|
|
|
||
|
|
if kind == "request_human_handoff":
|
||
|
|
async def request_handoff(
|
||
|
|
arguments: dict[str, Any],
|
||
|
|
) -> RealtimeToolResult:
|
||
|
|
reason = str(arguments.get("reason") or "human_handoff")
|
||
|
|
if not self._handoff_requested:
|
||
|
|
self._handoff_requested = True
|
||
|
|
self._store.values["system__handoff_status"] = "requested"
|
||
|
|
await self._runtime.queue_frame(
|
||
|
|
OutputTransportMessageUrgentFrame(
|
||
|
|
message={
|
||
|
|
"type": "handoff-requested",
|
||
|
|
"source": "prompt-system-tool",
|
||
|
|
"reason": reason,
|
||
|
|
"message": "用户请求转接人工服务。",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
)
|
||
|
|
await self._publish_variables(
|
||
|
|
"human_handoff",
|
||
|
|
["system__handoff_status"],
|
||
|
|
)
|
||
|
|
await self._refresh_instructions()
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{
|
||
|
|
"status": "requested",
|
||
|
|
"action": "human_handoff_requested",
|
||
|
|
"message": "人工接管请求已提交，请告知用户正在等待人工响应。",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
return (
|
||
|
|
RealtimeTool(
|
||
|
|
name=tool.function_name,
|
||
|
|
description=tool.description or "提交人工接管请求。",
|
||
|
|
properties={
|
||
|
|
"reason": {"type": "string", "description": "转接原因"}
|
||
|
|
},
|
||
|
|
),
|
||
|
|
request_handoff,
|
||
|
|
)
|
||
|
|
|
||
|
|
if kind == "end_conversation":
|
||
|
|
capture_reason = bool(config.get("capture_reason", True))
|
||
|
|
allow_end_interruption = bool(
|
||
|
|
config.get("allow_end_interruption", False)
|
||
|
|
)
|
||
|
|
message_type = str(config.get("message_type") or "none")
|
||
|
|
custom_message = str(config.get("custom_message") or "").strip()
|
||
|
|
|
||
|
|
async def end_conversation(
|
||
|
|
arguments: dict[str, Any],
|
||
|
|
) -> RealtimeToolResult:
|
||
|
|
if self._runtime.call_end.ending:
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{"status": "success", "action": "ending_call"},
|
||
|
|
follow_up="suppress",
|
||
|
|
)
|
||
|
|
reason = str(
|
||
|
|
arguments.get("reason") or "end_conversation"
|
||
|
|
).strip()
|
||
|
|
self._runtime.call_end.begin(
|
||
|
|
reason,
|
||
|
|
interruptible=allow_end_interruption,
|
||
|
|
)
|
||
|
|
if not allow_end_interruption:
|
||
|
|
self._set_input_enabled(False, force=True)
|
||
|
|
uses_custom_message = (
|
||
|
|
message_type == "custom" and bool(custom_message)
|
||
|
|
)
|
||
|
|
if uses_custom_message:
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{"status": "success", "action": "ending_call"},
|
||
|
|
follow_up="suppress",
|
||
|
|
after_output=lambda: self._play_end_message(custom_message),
|
||
|
|
)
|
||
|
|
self._waiting_generated_end = True
|
||
|
|
return RealtimeToolResult(
|
||
|
|
{
|
||
|
|
"status": "success",
|
||
|
|
"action": "ending_call",
|
||
|
|
"message": "请用一句简短、礼貌的结束语结束对话。",
|
||
|
|
}
|
||
|
|
)
|
||
|
|
|
||
|
|
properties = (
|
||
|
|
{
|
||
|
|
"reason": {
|
||
|
|
"type": "string",
|
||
|
|
"description": "结束本次通话的简短原因。",
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if capture_reason
|
||
|
|
else {}
|
||
|
|
)
|
||
|
|
return (
|
||
|
|
RealtimeTool(
|
||
|
|
name=tool.function_name,
|
||
|
|
description=tool.description or "礼貌结束本次对话。",
|
||
|
|
properties=properties,
|
||
|
|
required=("reason",) if capture_reason else (),
|
||
|
|
),
|
||
|
|
end_conversation,
|
||
|
|
)
|
||
|
|
return None
|
||
|
|
|
||
|
|
async def _play_end_message(self, content: str) -> None:
|
||
|
|
if not self._runtime.call_end.ending:
|
||
|
|
return
|
||
|
|
turn_id = uuid4().hex
|
||
|
|
timestamp = time_now_iso8601()
|
||
|
|
for message in (
|
||
|
|
{"type": "assistant-text-start", "turn_id": turn_id, "timestamp": timestamp},
|
||
|
|
{"type": "assistant-text-delta", "turn_id": turn_id, "delta": content},
|
||
|
|
{
|
||
|
|
"type": "assistant-text-end",
|
||
|
|
"turn_id": turn_id,
|
||
|
|
"content": content,
|
||
|
|
"interrupted": False,
|
||
|
|
},
|
||
|
|
):
|
||
|
|
await self._runtime.queue_frame(
|
||
|
|
OutputTransportMessageUrgentFrame(message=message)
|
||
|
|
)
|
||
|
|
track_speech = getattr(self._runtime.call_end, "track_speech", None)
|
||
|
|
playback_completion = track_speech() if callable(track_speech) else None
|
||
|
|
provider_completion = await self._runtime.realtime.speak_fixed(
|
||
|
|
content,
|
||
|
|
suppress_transcript=True,
|
||
|
|
)
|
||
|
|
marker = playback_marker_for(playback_completion)
|
||
|
|
if marker is not None:
|
||
|
|
if provider_completion is not None:
|
||
|
|
await provider_completion
|
||
|
|
await self._runtime.queue_frame(marker)
|
||
|
|
marker.completion.mark_queued()
|
||
|
|
await self._runtime.call_end.arm_after_tracked_speech()
|
||
|
|
await playback_completion
|
||
|
|
return
|
||
|
|
self._runtime.call_end.arm_after_speech()
|
||
|
|
|
||
|
|
async def _refresh_instructions(self) -> None:
|
||
|
|
await self._runtime.realtime.update_instructions(
|
||
|
|
self._store.render(self._cfg.prompt)
|
||
|
|
)
|
||
|
|
|
||
|
|
async def _publish_variables(self, reason: str, changed: list[str]) -> None:
|
||
|
|
await self._runtime.queue_frame(
|
||
|
|
OutputTransportMessageUrgentFrame(
|
||
|
|
message={
|
||
|
|
"type": "session-variables",
|
||
|
|
"reason": reason,
|
||
|
|
"variables": self._store.public_values(),
|
||
|
|
"changed": changed,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
)
|
||
|
|
|
||
|
|
async def _emit_trace(self, event: str, **details: Any) -> None:
|
||
|
|
try:
|
||
|
|
await self._runtime.queue_frame(
|
||
|
|
OutputTransportMessageUrgentFrame(
|
||
|
|
message={
|
||
|
|
"type": "workflow-event",
|
||
|
|
"eventId": f"wfe_{uuid4().hex[:20]}",
|
||
|
|
"event": event,
|
||
|
|
"timestamp": time_now_iso8601(),
|
||
|
|
"sessionId": self._runtime.session_id,
|
||
|
|
"source": "prompt-realtime",
|
||
|
|
**details,
|
||
|
|
}
|
||
|
|
)
|
||
|
|
)
|
||
|
|
except Exception as exc: # noqa: BLE001 - trace must not break tools
|
||
|
|
logger.warning(f"发送 Prompt Realtime 工具轨迹失败:{exc}")
|
||
|
|
|
||
|
|
def _lock_input(self) -> None:
|
||
|
|
self._input_locks += 1
|
||
|
|
self._set_input_enabled(False)
|
||
|
|
|
||
|
|
def _unlock_input(self) -> None:
|
||
|
|
self._input_locks = max(0, self._input_locks - 1)
|
||
|
|
if self._input_locks == 0 and not self._runtime.call_end.ending:
|
||
|
|
self._set_input_enabled(True)
|
||
|
|
|
||
|
|
def _set_input_enabled(self, enabled: bool, *, force: bool = False) -> None:
|
||
|
|
if self._runtime.set_input_enabled is None:
|
||
|
|
return
|
||
|
|
if force or enabled is False or self._input_locks == 0:
|
||
|
|
self._runtime.set_input_enabled(enabled)
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _public_error(exc: Exception) -> str:
|
||
|
|
message = str(exc).strip() or type(exc).__name__
|
||
|
|
return message[:MAX_PUBLIC_TOOL_ERROR]
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _public_tool_error(exc: Exception) -> str:
|
||
|
|
"""Return actionable errors without exposing URLs, payloads, or secrets."""
|
||
|
|
|
||
|
|
message = str(exc).strip()
|
||
|
|
if "超时" in message:
|
||
|
|
return "工具调用超时"
|
||
|
|
if message.startswith("客户端未注册工具"):
|
||
|
|
return message[:MAX_PUBLIC_TOOL_ERROR]
|
||
|
|
if any(token in message for token in ("参数", "变量", "绑定")):
|
||
|
|
return message[:MAX_PUBLIC_TOOL_ERROR]
|
||
|
|
return "工具调用失败，请稍后重试"
|