diff --git a/backend/db/models.py b/backend/db/models.py index acba2cc..478b0aa 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -10,6 +10,7 @@ from datetime import datetime from sqlalchemy import ( + CheckConstraint, JSON, Boolean, DateTime, @@ -175,6 +176,7 @@ class Assistant(Base): # ---- 瘦类型专属字段(真列,稀疏:按 type 用其中几列) ---- prompt: Mapped[str] = mapped_column(Text, default="") # prompt / opencode dynamic_variable_definitions: Mapped[dict] = mapped_column(JSON, default=dict) + built_in_tools: Mapped[dict] = mapped_column(JSONB, default=dict) analysis_config: Mapped[dict] = mapped_column(JSONB, default=dict) api_url: Mapped[str] = mapped_column(String(512), default="") # dify / fastgpt / opencode api_key: Mapped[str] = mapped_column(String(512), default="") # dify / fastgpt / opencode(打码/哨兵) @@ -245,6 +247,10 @@ class Tool(Base): "remote_tool_name", name="uq_tools_mcp_server_remote_name", ), + CheckConstraint( + "type IN ('http', 'mcp', 'client')", + name="ck_tools_external_type", + ), ) id: Mapped[str] = mapped_column(String(40), primary_key=True) @@ -273,7 +279,7 @@ class Tool(Base): class AssistantToolBinding(Base): - """Prompt assistant to reusable tool many-to-many binding.""" + """Assistant reference to a reusable tool plus Prompt usage config.""" __tablename__ = "assistant_tool_bindings" @@ -288,6 +294,7 @@ class AssistantToolBinding(Base): primary_key=True, index=True, ) + config: Mapped[dict] = mapped_column(JSONB, default=dict) created_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() ) diff --git a/backend/db/session.py b/backend/db/session.py index 4cbdca0..c377b90 100644 --- a/backend/db/session.py +++ b/backend/db/session.py @@ -10,7 +10,6 @@ import json import settings from services.interface_catalog import INTERFACE_DEFINITIONS -from services.system_tools import SYSTEM_TOOL_SPECS from sqlalchemy import text from sqlalchemy.ext.asyncio import ( AsyncSession, @@ -43,39 +42,19 @@ async def sync_interface_definitions() -> None: "interface_type": definition["interface_type"], "name": definition["name"], "capability": definition["capability"], - "field_schema": json.dumps({"fields": definition["fields"]}), + "field_schema": json.dumps( + { + "fields": definition["fields"], + "features": definition.get("features", {}), + } + ), }, ) async def sync_default_tools() -> None: - """Ensure system-provided reusable tools exist without overwriting edits.""" - default_system_tools = [] - for kind, spec in SYSTEM_TOOL_SPECS.items(): - config: dict[str, object] = {"kind": kind} - if kind == "end_conversation": - config.update( - { - "message_type": "none", - "custom_message": "", - "capture_reason": True, - "allow_end_interruption": False, - } - ) - default_system_tools.append( - { - **spec, - "type": "system", - "definition": { - "schema_version": 1, - "type": "system", - "config": config, - }, - } - ) - + """Ensure workspace-owned reusable external tools exist.""" default_tools = [ - *default_system_tools, { "id": "tool_show_message_default", "name": "显示确认消息", @@ -83,7 +62,7 @@ async def sync_default_tools() -> None: "type": "client", "description": "向用户显示必须主动确认的重要消息弹窗。", "definition": { - "schema_version": 1, + "schema_version": 2, "type": "client", "config": { "allow_interruptions": False, @@ -121,7 +100,6 @@ async def sync_default_tools() -> None: "required": False, }, ], - "dynamic_variable_assignments": {}, }, }, }, diff --git a/backend/migrations/versions/20260813_0021_organize_tool_resources.py b/backend/migrations/versions/20260813_0021_organize_tool_resources.py new file mode 100644 index 0000000..a2d4b1f --- /dev/null +++ b/backend/migrations/versions/20260813_0021_organize_tool_resources.py @@ -0,0 +1,510 @@ +"""separate built-ins, resources, and per-use tool bindings + +Revision ID: 20260813_0021 +Revises: 20260812_0020 +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import deepcopy +import json +import re + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + + +revision: str = "20260813_0021" +down_revision: str | Sequence[str] | None = "20260812_0020" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +PLACEHOLDER = re.compile(r"{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}") +FULL_PLACEHOLDER = re.compile(r"^{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}$") + + +def _source_for(value) -> dict: + raw = json.dumps(value, ensure_ascii=False) + return ( + {"source": "template", "value": value} + if PLACEHOLDER.search(raw) + else {"source": "literal", "value": value} + ) + + +def _llm_rule(name: str, definition: Mapping | None) -> dict: + value_type = str((definition or {}).get("type") or "string") + return { + "variableName": name, + "expression": { + "type": "llm", + "parameterName": name, + "valueType": value_type, + "description": f"从当前对话中提取并确认 {name}", + }, + } + + +def _builtin( + kind: str, + tool: Mapping, + definitions: Mapping, + allowed=None, + *, + camel_case: bool = False, +) -> dict: + definition = tool.get("definition") or {} + config = deepcopy(definition.get("config") or {}) + config.pop("kind", None) + config["description"] = str(tool.get("description") or "") + if kind == "update_state": + names = list(allowed) if allowed is not None else list(definitions) + config["updates"] = [ + _llm_rule(str(name), definitions.get(str(name))) + for name in names + if str(name) in definitions + ] + if camel_case: + aliases = { + "message_type": "messageType", + "custom_message": "customMessage", + "capture_reason": "captureReason", + "allow_end_interruption": "allowEndInterruption", + } + config = {aliases.get(key, key): value for key, value in config.items()} + return config + + +def _builtin_key(kind: str) -> str: + return { + "end_conversation": "endConversation", + "update_state": "updateState", + "skip_turn": "skipTurn", + "request_human_handoff": "requestHumanHandoff", + }.get(kind, kind) + + +def _binding( + tool_id: str, + tool: Mapping, + arguments=None, + assignments=None, + inherited_assignments: Mapping | None = None, +) -> dict: + config = ((tool.get("definition") or {}).get("config") or {}) + if tool.get("type") == "mcp": + schema = config.get("input_schema") or {} + names = list((schema.get("properties") or {}).keys()) + else: + names = [ + str(parameter.get("name")) + for parameter in config.get("parameters") or [] + if parameter.get("name") + ] + if arguments is None: + argument_bindings = {name: {"source": "model"} for name in names} + else: + argument_bindings = { + str(name): _source_for(value) for name, value in arguments.items() + } + return { + "toolId": tool_id, + "argumentBindings": argument_bindings, + "templateBindings": {}, + "resultAssignments": dict( + (inherited_assignments or {}) if assignments is None else assignments + ), + } + + +def _state_expression(value) -> dict: + """Translate deterministic v3 values without silently changing semantics.""" + if isinstance(value, str): + match = FULL_PLACEHOLDER.fullmatch(value) + if match: + return {"type": "variable", "variableName": match.group(1)} + if PLACEHOLDER.search(value): + raise RuntimeError( + "Update State 节点中的混合字符串模板无法无损迁移；" + "请在升级前改成完整变量占位符" + ) + if not isinstance(value, (str, int, float, bool)): + raise RuntimeError("Update State 节点只支持字符串、数字、布尔值或变量") + return {"type": "constant", "value": value} + + +def _rewrite_templates(value, inputs: set[str]): + if isinstance(value, str): + def replace(match): + name = match.group(1) + if name.startswith(("system__", "secret__")): + return match.group(0) + if name.startswith("input__"): + inputs.add(name.removeprefix("input__")) + return match.group(0) + inputs.add(name) + return "{{input__" + name + "}}" + return PLACEHOLDER.sub(replace, value) + if isinstance(value, list): + return [_rewrite_templates(item, inputs) for item in value] + if isinstance(value, dict): + return {key: _rewrite_templates(item, inputs) for key, item in value.items()} + return value + + +def _add_template_bindings(binding: dict, inputs: set[str]) -> None: + binding.setdefault("templateBindings", {}).update( + { + name: {"source": "variable", "variableName": name} + for name in sorted(inputs) + } + ) + + +def _freeze_inherited_agent_for_update_state( + data: dict, + settings: Mapping, + definitions: Mapping, + allowed: list, +) -> None: + """Preserve the old per-Agent Update State write scope. + + Before v4 an Agent could inherit the global Update State resource while its + writable variable names still lived on the node. A v4 built-in owns its + complete configuration, so this one exceptional case must snapshot the + current global stage configuration onto the Agent. + """ + + runtime_mode = str(settings.get("runtimeMode") or "pipeline") + if runtime_mode != "realtime": + for source, target in ( + ("defaultLlmResourceId", "llmResourceId"), + ("defaultAsrResourceId", "asrResourceId"), + ("defaultTtsResourceId", "ttsResourceId"), + ): + data[target] = settings.get(source) + for key in ("enableInterrupt", "turnConfig"): + if key in settings: + data[key] = deepcopy(settings[key]) + for key in ( + "visionEnabled", + "visionModelResourceId", + "visionPromptMode", + "visionPrompt", + "knowledgeBaseId", + "knowledgeMode", + "knowledgeTopN", + "knowledgeScoreThreshold", + ): + if key in settings: + data[key] = deepcopy(settings[key]) + data["toolBindings"] = deepcopy(settings.get("toolBindings") or []) + data["builtInTools"] = deepcopy(settings.get("builtInTools") or {}) + data["builtInTools"]["updateState"]["updates"] = [ + _llm_rule(str(name), definitions.get(str(name))) + for name in allowed + if str(name) in definitions + ] + global_prompt = str(settings.get("globalPrompt") or "").strip() + node_prompt = str(data.get("prompt") or "").strip() + if global_prompt: + data["prompt"] = "\n\n".join(part for part in (global_prompt, node_prompt) if part) + data["inheritGlobalConfig"] = False + + +def upgrade() -> None: + connection = op.get_bind() + op.add_column( + "assistants", + sa.Column( + "built_in_tools", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + ) + op.add_column( + "assistant_tool_bindings", + sa.Column( + "config", + postgresql.JSONB(astext_type=sa.Text()), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + ) + + tool_rows = connection.execute( + sa.text("SELECT id, type, description, definition, mcp_server_id FROM tools") + ).mappings().all() + tools = {str(row["id"]): dict(row) for row in tool_rows} + template_inputs: dict[str, set[str]] = {} + resource_assignments: dict[str, dict] = {} + server_inputs: dict[str, set[str]] = {} + server_rows = connection.execute( + sa.text("SELECT id, url, config FROM mcp_servers") + ).mappings().all() + for row in server_rows: + inputs: set[str] = set() + config = deepcopy(row["config"] or {}) + url = _rewrite_templates(str(row["url"] or ""), inputs) + config["headers"] = _rewrite_templates(config.get("headers") or {}, inputs) + server_id = str(row["id"]) + server_inputs[server_id] = inputs + connection.execute( + sa.text( + "UPDATE mcp_servers SET url=:url, config=CAST(:config AS jsonb) " + "WHERE id=:id" + ), + {"id": server_id, "url": url, "config": json.dumps(config)}, + ) + for tool_id, tool in tools.items(): + if tool.get("type") == "system": + continue + definition = deepcopy(tool.get("definition") or {}) + config = definition.get("config") or {} + definition["config"] = config + resource_assignments[tool_id] = dict( + config.get("dynamic_variable_assignments") or {} + ) + config.pop("dynamic_variable_assignments", None) + inputs: set[str] = set() + if tool.get("type") == "http": + for key in ("url", "headers", "body"): + config[key] = _rewrite_templates(config.get(key), inputs) + config["template_inputs"] = [ + {"name": name, "type": "string", "location": "body", "required": True} + for name in sorted(inputs) + ] + elif tool.get("type") == "mcp": + inputs.update(server_inputs.get(str(tool.get("mcp_server_id") or ""), set())) + config["template_inputs"] = [ + { + "name": name, + "type": "string", + "location": "body", + "required": True, + } + for name in sorted(inputs) + ] + definition["schema_version"] = 2 + template_inputs[tool_id] = inputs + connection.execute( + sa.text("UPDATE tools SET definition=CAST(:definition AS jsonb) WHERE id=:id"), + {"id": tool_id, "definition": json.dumps(definition)}, + ) + tool["definition"] = definition + + assistant_rows = connection.execute( + sa.text( + "SELECT id, type, runtime_mode, graph, startup, " + "dynamic_variable_definitions FROM assistants" + ) + ).mappings().all() + for assistant in assistant_rows: + assistant_id = str(assistant["id"]) + definitions = assistant["dynamic_variable_definitions"] or {} + binding_rows = connection.execute( + sa.text( + "SELECT tool_id FROM assistant_tool_bindings " + "WHERE assistant_id=:assistant_id ORDER BY created_at, tool_id" + ), + {"assistant_id": assistant_id}, + ).mappings().all() + selected_ids = [str(row["tool_id"]) for row in binding_rows] + system_by_kind: dict[str, Mapping] = {} + for tool_id in selected_ids: + tool = tools.get(tool_id) or {} + kind = ((tool.get("definition") or {}).get("config") or {}).get("kind") + if tool.get("type") == "system" and kind: + if kind in system_by_kind: + raise RuntimeError( + f"Assistant {assistant_id} 同时绑定多个 {kind} System Tool" + ) + system_by_kind[str(kind)] = tool + + builtins = ( + { + kind: _builtin(kind, tool, definitions) + for kind, tool in system_by_kind.items() + } + if ( + assistant["type"] == "prompt" + and assistant["runtime_mode"] != "realtime" + ) + else {} + ) + for tool_id in selected_ids: + tool = tools.get(tool_id) or {} + if tool.get("type") == "system": + continue + binding = _binding( + tool_id, + tool, + inherited_assignments=resource_assignments.get(tool_id), + ) + _add_template_bindings(binding, template_inputs.get(tool_id, set())) + connection.execute( + sa.text( + "UPDATE assistant_tool_bindings SET config=CAST(:config AS jsonb) " + "WHERE assistant_id=:assistant_id AND tool_id=:tool_id" + ), + { + "assistant_id": assistant_id, + "tool_id": tool_id, + "config": json.dumps( + { + "argument_bindings": binding["argumentBindings"], + "template_bindings": binding["templateBindings"], + "result_assignments": binding["resultAssignments"], + } + ), + }, + ) + + startup = deepcopy(assistant["startup"] or {}) + for action in startup.get("actions") or []: + tool_id = str(action.pop("toolId", action.pop("tool_id", "")) or "") + arguments = action.pop("arguments", {}) or {} + tool = tools.get(tool_id) or {} + binding = _binding( + tool_id, + tool, + arguments=arguments, + inherited_assignments=resource_assignments.get(tool_id), + ) + _add_template_bindings(binding, template_inputs.get(tool_id, set())) + action["toolBinding"] = binding + + graph = deepcopy(assistant["graph"] or {}) + if assistant["type"] == "workflow": + graph["specVersion"] = 4 + settings = graph.setdefault("settings", {}) + global_ids = list(settings.pop("toolIds", []) or []) + settings["toolBindings"] = [] + settings["builtInTools"] = {} + for tool_id in global_ids: + tool_id = str(tool_id) + tool = tools.get(tool_id) or {} + if tool.get("type") == "system": + kind = ((tool.get("definition") or {}).get("config") or {}).get("kind") + if kind: + settings["builtInTools"][_builtin_key(str(kind))] = _builtin( + str(kind), tool, definitions, camel_case=True + ) + continue + binding = _binding( + tool_id, + tool, + inherited_assignments=resource_assignments.get(tool_id), + ) + _add_template_bindings(binding, template_inputs.get(tool_id, set())) + settings["toolBindings"].append(binding) + for node in graph.get("nodes") or []: + data = node.setdefault("data", {}) + if node.get("type") == "agent": + inherits_global = bool(data.get("inheritGlobalConfig", True)) + ids = list(data.pop("toolIds", []) or []) + allowed = list(data.pop("stateVariableNames", []) or []) + data["toolBindings"] = [] + data["builtInTools"] = {} + if inherits_global: + if "updateState" in settings["builtInTools"]: + _freeze_inherited_agent_for_update_state( + data, settings, definitions, allowed + ) + else: + for tool_id in ids: + tool_id = str(tool_id) + tool = tools.get(tool_id) or {} + if tool.get("type") == "system": + kind = ((tool.get("definition") or {}).get("config") or {}).get("kind") + if kind: + data["builtInTools"][_builtin_key(str(kind))] = _builtin( + str(kind), + tool, + definitions, + allowed if kind == "update_state" else None, + camel_case=True, + ) + continue + binding = _binding( + tool_id, + tool, + inherited_assignments=resource_assignments.get(tool_id), + ) + _add_template_bindings( + binding, template_inputs.get(tool_id, set()) + ) + data["toolBindings"].append(binding) + elif node.get("type") == "action": + tool_id = str(data.pop("toolId", "") or "") + arguments = data.pop("arguments", {}) or {} + mode = str(data.pop("resultAssignmentMode", "none") or "none") + overrides = data.pop("resultAssignments", {}) or {} + assignments = ( + None if mode == "inherit" else overrides if mode == "override" else {} + ) + binding = _binding( + tool_id, + tools.get(tool_id) or {}, + arguments, + assignments, + resource_assignments.get(tool_id), + ) + _add_template_bindings(binding, template_inputs.get(tool_id, set())) + data["toolBinding"] = binding + elif node.get("type") == "update_state": + assignments = data.pop("assignments", {}) or {} + data["updates"] = [ + { + "variableName": str(name), + "expression": _state_expression(value), + } + for name, value in assignments.items() + ] + + connection.execute( + sa.text( + "UPDATE assistants SET built_in_tools=CAST(:builtins AS jsonb), " + "startup=CAST(:startup AS json), graph=CAST(:graph AS json) WHERE id=:id" + ), + { + "id": assistant_id, + "builtins": json.dumps(builtins), + "startup": json.dumps(startup), + "graph": json.dumps(graph), + }, + ) + + if assistant["type"] == "workflow": + connection.execute( + sa.text( + "UPDATE assistant_tool_bindings SET config='{}'::jsonb " + "WHERE assistant_id=:assistant_id" + ), + {"assistant_id": assistant_id}, + ) + + connection.execute( + sa.text( + "DELETE FROM assistant_tool_bindings " + "WHERE tool_id IN (SELECT id FROM tools WHERE type='system')" + ) + ) + connection.execute( + sa.text("DELETE FROM tools WHERE type='system'") + ) + op.create_check_constraint( + "ck_tools_external_type", + "tools", + "type IN ('http', 'mcp', 'client')", + ) + + +def downgrade() -> None: + raise RuntimeError( + "Tool Resource v4 是一次性边界迁移；请从部署前数据库备份回滚。" + ) diff --git a/backend/migrations/versions/20260813_0022_clean_realtime_prompt_builtins.py b/backend/migrations/versions/20260813_0022_clean_realtime_prompt_builtins.py new file mode 100644 index 0000000..abe7727 --- /dev/null +++ b/backend/migrations/versions/20260813_0022_clean_realtime_prompt_builtins.py @@ -0,0 +1,38 @@ +"""remove unsupported built-ins from realtime prompt assistants + +Revision ID: 20260813_0022 +Revises: 20260813_0021 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260813_0022" +down_revision: str | Sequence[str] | None = "20260813_0021" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Make stored configuration match the current Realtime Prompt runtime.""" + + op.get_bind().execute( + sa.text( + "UPDATE assistants " + "SET built_in_tools = '{}'::jsonb " + "WHERE type = 'prompt' " + "AND runtime_mode = 'realtime' " + "AND built_in_tools <> '{}'::jsonb" + ) + ) + + +def downgrade() -> None: + # The removed values cannot be reconstructed and were never executable in + # Realtime Prompt sessions. Keep downgrade structurally safe and explicit. + pass diff --git a/backend/models.py b/backend/models.py index 1fb0bee..9eb22ff 100644 --- a/backend/models.py +++ b/backend/models.py @@ -10,7 +10,8 @@ from __future__ import annotations from typing import Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field +from pydantic.alias_generators import to_camel RuntimeMode = Literal["pipeline", "realtime"] VisionPromptMode = Literal["fixed", "runtime", "guided"] @@ -42,6 +43,17 @@ class RuntimeTool(BaseModel): mcp_server: RuntimeMcpServer | None = None +class RuntimeToolBinding(BaseModel): + """One assistant/stage-specific use of a reusable external tool.""" + + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) + + tool_id: str + argument_bindings: dict = Field(default_factory=dict) + template_bindings: dict = Field(default_factory=dict) + result_assignments: dict[str, str] = Field(default_factory=dict) + + class RuntimeModelResource(BaseModel): id: str name: str = "" @@ -112,13 +124,12 @@ class AssistantConfig(BaseModel): turnConfig: dict = Field(default_factory=dict) startup: dict = Field(default_factory=dict) analysis_config: dict = Field(default_factory=dict) + built_in_tools: dict = Field(default_factory=dict) - # ``tools`` is the complete runtime pool (conversation + lifecycle actions). - # ``llm_tool_ids`` limits which tools are advertised to a Prompt model. None - # preserves compatibility for inline/test configs that historically exposed - # every item in ``tools``. + # ``tools`` is the reusable resource pool. Bindings define per-use inputs + # and result mappings; platform built-ins are configured separately. tools: list[RuntimeTool] = Field(default_factory=list) - llm_tool_ids: list[str] | None = None + tool_bindings: list[RuntimeToolBinding] = Field(default_factory=list) knowledge_base_id: str | None = None knowledge_base_name: str = "" knowledge_base_description: str = "" @@ -168,3 +179,5 @@ class SignalingOffer(BaseModel): inline_config: AssistantConfig | None = None vision_enabled: bool = False dynamic_variables: dict = Field(default_factory=dict) + client_tool_names: list[str] = Field(default_factory=list) + model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) diff --git a/backend/routes/assistants.py b/backend/routes/assistants.py index 775f623..ae803bf 100644 --- a/backend/routes/assistants.py +++ b/backend/routes/assistants.py @@ -2,7 +2,7 @@ import uuid -from models import VISION_PROMPT_MODES, RuntimeTool +from models import VISION_PROMPT_MODES, RuntimeTool, RuntimeToolBinding from db.models import ( Assistant, AssistantModelBinding, @@ -10,6 +10,7 @@ from db.models import ( KnowledgeBase, ModelResource, Tool, + InterfaceDefinition, ) from db.session import get_session from fastapi import APIRouter, Depends, HTTPException @@ -20,8 +21,10 @@ from schemas import ( from services.auth import require_admin from services.masking import mask, resolve_incoming_key from services.node_specs import graph_references, normalize_graph, validate_graph -from services.system_tools import system_tool_kind +from services.state_expressions import StateExpressionError, validate_update_rules +from services.system_tools import RESERVED_TOOL_NAMES from services.tool_policy import policy_for_tool +from services.tool_executor import ToolExecutor from services.workflow_engine import WorkflowEngine from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -34,10 +37,69 @@ router = APIRouter( CAPABILITIES = ("LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent") +async def _validate_prompt_realtime_tool_capabilities( + session: AsyncSession, + body: AssistantUpsert, +) -> None: + """Gate Prompt tools by the selected realtime interface definition.""" + + if body.type != "prompt" or body.runtime_mode != "realtime": + return + resource_id = str(body.model_resource_ids.get("Realtime") or "") + resource = await session.get(ModelResource, resource_id) if resource_id else None + if not resource or not resource.enabled or resource.capability != "Realtime": + raise HTTPException(400, "Prompt Realtime 必须选择可用的 Realtime 模型") + definition = await session.get(InterfaceDefinition, resource.interface_type) + features = ( + ((definition.field_schema or {}).get("features") or {}).get( + "realtimePromptTools" + ) + if definition and definition.enabled + else None + ) or {} + external_types = set(features.get("externalTypes") or []) + built_ins = set(features.get("builtIns") or []) + tool_ids = [binding.tool_id for binding in body.tool_bindings] + rows = ( + ( + await session.execute(select(Tool).where(Tool.id.in_(tool_ids))) + ).scalars().all() + if tool_ids + else [] + ) + unsupported_external = sorted( + {tool.type for tool in rows if tool.type not in external_types} + ) + requested_built_ins = { + key + for key, value in body.built_in_tools.model_dump( + by_alias=True, + exclude_none=True, + ).items() + if value is not None + } + unsupported_built_ins = sorted(requested_built_ins - built_ins) + if unsupported_external or unsupported_built_ins: + details = [] + if unsupported_external: + details.append("外部工具:" + ",".join(unsupported_external)) + if unsupported_built_ins: + details.append("内建工具:" + ",".join(unsupported_built_ins)) + raise HTTPException( + 400, + f"Realtime 接口 {resource.interface_type} 不支持 " + "；".join(details), + ) + + def _validate_workflow(body: AssistantUpsert) -> None: """workflow 类型:保存前校验图结构,不通过则 400。其他类型跳过。""" if body.type != "workflow": return + if body.graph and str(body.graph.get("specVersion") or "") != "4": + raise HTTPException( + 400, + "Workflow API 仅接受 specVersion=4；旧图请先执行数据库迁移", + ) body.graph = normalize_graph(body.graph or {}) errors = validate_graph(body.graph) settings = body.graph.get("settings") or {} @@ -48,28 +110,6 @@ def _validate_workflow(body: AssistantUpsert) -> None: bound_realtime_id = str(body.model_resource_ids.get("Realtime") or "") if graph_realtime_id != bound_realtime_id: errors.append("工作流 Realtime 模型与 Assistant Realtime 绑定不一致") - declared_variables = set(body.dynamic_variable_definitions) - for node in body.graph.get("nodes") or []: - node_id = str(node.get("id") or "") - node_type = node.get("type") - data = node.get("data") or {} - if node_type == "update_state": - unknown = sorted( - set((data.get("assignments") or {}).keys()) - declared_variables - ) - if unknown: - errors.append( - f"Update State 节点 {node_id} 引用了未声明变量:" - + ",".join(unknown) - ) - elif node_type == "agent": - authorized = set(data.get("stateVariableNames") or []) - unknown = sorted(authorized - declared_variables) - if unknown: - errors.append( - f"Agent 节点 {node_id} 授权了未声明变量:" - + ",".join(unknown) - ) if errors: raise HTTPException(400, "工作流校验失败:" + ";".join(errors)) # Graph settings are the source of truth. The flat flag is only a session @@ -78,8 +118,6 @@ def _validate_workflow(body: AssistantUpsert) -> None: body.vision_model_resource_id = None body.vision_prompt_mode = "runtime" body.vision_prompt = "" - refs = graph_references(body.graph) - body.tool_ids = list(dict.fromkeys([*body.tool_ids, *sorted(refs["tools"])])) async def _validate_workflow_references( @@ -156,58 +194,202 @@ async def _validate_workflow_references( raise HTTPException(400, f"Workflow 引用了无效知识库:{knowledge_id}") -async def _validate_system_tool_selection( +async def _validate_tool_bindings( session: AsyncSession, body: AssistantUpsert, ) -> None: - """Validate System resources in the same selection path as every other tool.""" - selected_ids = list(dict.fromkeys(body.tool_ids)) - if not selected_ids or body.type not in {"prompt", "workflow"}: - return - rows = ( - await session.execute(select(Tool).where(Tool.id.in_(selected_ids))) - ).scalars().all() - tools_by_id = {tool.id: tool for tool in rows if tool.status == "active"} - system_kinds = { - tool.id: system_tool_kind(tool.definition or {}) - for tool in rows - if tool.status == "active" and tool.type == "system" - } - invalid = [tool_id for tool_id, kind in system_kinds.items() if kind is None] - if invalid: - raise HTTPException(400, "系统工具配置无效: " + ", ".join(invalid)) + """Validate reusable resources and every usage-specific binding.""" + raw_bindings: list[tuple[str, dict]] = [] if body.type == "prompt": - if body.runtime_mode == "realtime" and system_kinds: - raise HTTPException(400, "Prompt Realtime 模式暂不支持系统工具") - if ( - "update_state" in system_kinds.values() - and not body.dynamic_variable_definitions - ): - raise HTTPException(400, "启用更新状态工具前必须声明至少一个动态变量") - return - - engine = WorkflowEngine(body.graph) - for node_id, node in engine.nodes.items(): - node_type = node.get("type") - data = node.get("data") or {} - if node_type == "agent": - stage = engine.agent_stage_config(node_id) - kinds = { - system_kinds[tool_id] - for tool_id in stage.tool_ids - if tool_id in system_kinds - } - if "update_state" in kinds and not stage.state_variable_names: + raw_bindings.extend( + ("Prompt", binding.model_dump()) for binding in body.tool_bindings + ) + for action in body.startup.actions: + raw_bindings.append( + (f"Startup {action.id}", action.tool_binding.model_dump()) + ) + elif body.type == "workflow": + settings = body.graph.get("settings") or {} + raw_bindings.extend( + ("Workflow 全局", binding) + for binding in settings.get("toolBindings") or [] + ) + for node in body.graph.get("nodes") or []: + data = node.get("data") or {} + if node.get("type") == "agent" and not data.get( + "inheritGlobalConfig", True + ): + raw_bindings.extend( + (f"Agent {node.get('id')}", binding) + for binding in data.get("toolBindings") or [] + ) + elif node.get("type") == "action": + raw_bindings.append( + (f"Action {node.get('id')}", data.get("toolBinding") or {}) + ) + tool_ids = { + str(binding.get("tool_id") or binding.get("toolId") or "") + for _, binding in raw_bindings + if isinstance(binding, dict) + } + tool_ids.discard("") + rows = ( + await session.execute(select(Tool).where(Tool.id.in_(tool_ids))) + ).scalars().all() if tool_ids else [] + tools = {tool.id: tool for tool in rows if tool.status == "active"} + missing = sorted(tool_ids - set(tools)) + if missing: + raise HTTPException(400, "工具不存在或未启用: " + ", ".join(missing)) + reserved = sorted( + tool.function_name + for tool in tools.values() + if tool.function_name in RESERVED_TOOL_NAMES + ) + if reserved: + raise HTTPException( + 400, + "外部工具使用了平台保留函数名: " + ", ".join(reserved), + ) + declared_variables = set(body.dynamic_variable_definitions) + system_variables = { + "system__agent_id", + "system__conversation_id", + "system__time", + "system__time_utc", + "system__timezone", + "system__agent_turns", + "system__conversation_history", + "system__last_action_status", + "system__last_action_invocation_id", + "system__last_action_duration_ms", + "system__last_action_error", + "system__last_action_error_code", + "system__handoff_status", + } + for scope, raw in raw_bindings: + if not isinstance(raw, dict): + raise HTTPException(400, "工具绑定必须是对象") + binding = RuntimeToolBinding(**raw) + tool = tools.get(binding.tool_id) + if tool is None: + continue + runtime_tool = RuntimeTool( + id=tool.id, + name=tool.name, + function_name=tool.function_name, + type=tool.type, + definition=tool.definition or {}, + ) + properties, required = ToolExecutor.input_schema(runtime_tool) + unknown = sorted(set(binding.argument_bindings) - set(properties)) + if unknown: + raise HTTPException(400, "工具绑定引用了未知参数: " + ", ".join(unknown)) + absent = sorted(set(required) - set(binding.argument_bindings)) + if absent: + raise HTTPException(400, "工具绑定缺少必填参数: " + ", ".join(absent)) + if scope.startswith(("Action ", "Startup ")): + model_parameters = sorted( + name + for name, source in binding.argument_bindings.items() + if str((source or {}).get("source") or "") == "model" + ) + if model_parameters: raise HTTPException( 400, - f"Agent 节点 {node_id} 启用更新状态工具时必须授权至少一个变量", + f"{scope} 是确定性调用，参数不能来自模型: " + + ", ".join(model_parameters), ) - elif node_type == "action": - tool_id = str(data.get("toolId") or "") - tool = tools_by_id.get(tool_id) - if tool and tool.type == "system": - raise HTTPException(400, f"Action 节点 {node_id} 不能调用系统工具") + variable_sources = { + str(source.get("variable_name") or source.get("variableName") or "") + for source in [ + *binding.argument_bindings.values(), + *binding.template_bindings.values(), + ] + if str((source or {}).get("source") or "") == "variable" + } + invalid_sources = sorted(variable_sources - declared_variables - system_variables) + if invalid_sources: + raise HTTPException( + 400, + f"{scope} 引用了未声明动态变量: " + ", ".join(invalid_sources), + ) + config = (tool.definition or {}).get("config") or {} + if tool.type == "client" and not config.get( + "wait_for_response", config.get("waitForResponse", True) + ) and binding.result_assignments: + raise HTTPException( + 400, + f"{scope} 的 fire-and-forget Client 工具不能配置结果映射", + ) + template_inputs = config.get("template_inputs") or [] + template_names = { + str(item.get("name")) for item in template_inputs if item.get("name") + } + unknown_templates = sorted(set(binding.template_bindings) - template_names) + missing_templates = sorted( + { + str(item.get("name")) + for item in template_inputs + if item.get("name") and item.get("required", True) + } + - set(binding.template_bindings) + ) + if unknown_templates: + raise HTTPException( + 400, + "工具绑定引用了未知模板输入: " + ", ".join(unknown_templates), + ) + if missing_templates: + raise HTTPException( + 400, + "工具绑定缺少模板输入: " + ", ".join(missing_templates), + ) + undeclared = sorted( + set(binding.result_assignments) - set(body.dynamic_variable_definitions) + ) + if undeclared: + raise HTTPException(400, "结果映射目标变量未声明: " + ", ".join(undeclared)) + + +def _validate_update_state_config(body: AssistantUpsert) -> None: + """Validate Prompt and Workflow built-ins against declared variables.""" + + configs: list[tuple[str, dict]] = [] + if body.type == "prompt": + configs.append(("Prompt", body.built_in_tools.model_dump())) + elif body.type == "workflow": + settings = body.graph.get("settings") or {} + configs.append(("Workflow 全局", settings.get("builtInTools") or {})) + for node in body.graph.get("nodes") or []: + data = node.get("data") or {} + if node.get("type") == "agent" and not data.get( + "inheritGlobalConfig", True + ): + configs.append( + (f"Agent {node.get('id')}", data.get("builtInTools") or {}) + ) + elif node.get("type") == "update_state": + try: + validate_update_rules( + data.get("updates") or [], + body.dynamic_variable_definitions, + allow_llm=False, + ) + except StateExpressionError as exc: + raise HTTPException(400, f"Update State 节点配置无效: {exc}") from exc + for scope, config in configs: + update_state = config.get("update_state") or config.get("updateState") + if not update_state: + continue + try: + validate_update_rules( + update_state.get("updates") or [], + body.dynamic_variable_definitions, + allow_llm=True, + ) + except StateExpressionError as exc: + raise HTTPException(400, f"{scope} 更新状态配置无效: {exc}") from exc async def _validate_startup_actions( @@ -220,9 +402,10 @@ async def _validate_startup_actions( if not actions: return for action in actions: - tool = await session.get(Tool, action.tool_id) + binding = action.tool_binding + tool = await session.get(Tool, binding.tool_id) if not tool or tool.status != "active": - raise HTTPException(400, f"启动 Action 引用了无效工具:{action.tool_id}") + raise HTTPException(400, f"启动 Action 引用了无效工具:{binding.tool_id}") runtime_tool = RuntimeTool( id=tool.id, name=tool.name, @@ -247,8 +430,13 @@ async def _validate_startup_actions( continue if policy.response_wait_mode != "session": raise HTTPException(400, "show_message 启动 Action 必须使用会话内等待") - message = str(action.arguments.get("message") or "").strip() - buttons = action.arguments.get("actions") + literal_arguments = { + name: source.value + for name, source in binding.argument_bindings.items() + if getattr(source, "source", "") == "literal" + } + message = str(literal_arguments.get("message") or "").strip() + buttons = literal_arguments.get("actions") if not message or len(message) > 2000: raise HTTPException(400, "show_message 重要信息必须为 1-2000 个字符") if not isinstance(buttons, list) or not buttons: @@ -258,7 +446,7 @@ async def _validate_startup_actions( first_button.get("label") or "" ).strip(): raise HTTPException(400, "show_message 确认按钮必须配置 id 和文字") - if action.required and action.arguments.get("dismissible") is not False: + if action.required and literal_arguments.get("dismissible") is not False: raise HTTPException(400, "必需的 show_message 启动 Action 不允许跳过确认") @@ -346,13 +534,30 @@ async def _resource_ids(session: AsyncSession, assistant_id: str) -> dict[str, s async def _sync_tool_bindings( - session: AsyncSession, assistant_id: str, assistant_type: str, tool_ids: list[str] + session: AsyncSession, + assistant_id: str, + assistant_type: str, + tool_bindings, + graph: dict | None = None, ) -> None: - requested = ( - list(dict.fromkeys(tool_ids)) - if assistant_type in {"prompt", "workflow"} - else [] - ) + requested_configs = { + binding.tool_id: { + "argument_bindings": { + name: source.model_dump() + for name, source in binding.argument_bindings.items() + }, + "template_bindings": { + name: source.model_dump() + for name, source in binding.template_bindings.items() + }, + "result_assignments": dict(binding.result_assignments), + } + for binding in tool_bindings + } if assistant_type == "prompt" else {} + if assistant_type == "workflow": + refs = graph_references(graph or {}) + requested_configs = {tool_id: {} for tool_id in refs["tools"]} + requested = list(requested_configs) if requested: tools = ( await session.execute(select(Tool).where(Tool.id.in_(requested))) @@ -374,19 +579,31 @@ async def _sync_tool_bindings( if tool_id not in requested: await session.delete(binding) for tool_id in requested: - if tool_id not in existing_by_id: - session.add(AssistantToolBinding(assistant_id=assistant_id, tool_id=tool_id)) + if tool_id in existing_by_id: + existing_by_id[tool_id].config = requested_configs[tool_id] + else: + session.add( + AssistantToolBinding( + assistant_id=assistant_id, + tool_id=tool_id, + config=requested_configs[tool_id], + ) + ) -async def _tool_ids(session: AsyncSession, assistant_id: str) -> list[str]: +async def _tool_bindings(session: AsyncSession, assistant_id: str) -> list[dict]: rows = ( await session.execute( - select(AssistantToolBinding.tool_id) + select(AssistantToolBinding) .where(AssistantToolBinding.assistant_id == assistant_id) .order_by(AssistantToolBinding.created_at) ) ).scalars().all() - return list(rows) + return [ + {"tool_id": row.tool_id, **(row.config or {})} + for row in rows + if row.config + ] async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut: @@ -406,7 +623,12 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut: model_resource_ids=await _resource_ids(session, assistant.id), knowledge_base_id=assistant.knowledge_base_id, knowledge_retrieval_config=assistant.knowledge_retrieval_config or {}, - tool_ids=await _tool_ids(session, assistant.id), + tool_bindings=( + await _tool_bindings(session, assistant.id) + if assistant.type == "prompt" + else [] + ), + built_in_tools=assistant.built_in_tools or {}, prompt=assistant.prompt, dynamic_variable_definitions=assistant.dynamic_variable_definitions or {}, analysis_config=assistant.analysis_config or {}, @@ -436,19 +658,29 @@ async def create_assistant( ): _validate_workflow(body) await _validate_workflow_references(session, body) - await _validate_system_tool_selection(session, body) + _validate_update_state_config(body) + await _validate_tool_bindings(session, body) + await _validate_prompt_realtime_tool_capabilities(session, body) await _validate_startup_actions(session, body) await _validate_vision_model(session, body) await _validate_analysis_model(session, body) await _validate_knowledge_base(session, body) data = body.model_dump() resource_ids = data.pop("model_resource_ids") - tool_ids = data.pop("tool_ids") + tool_bindings = body.tool_bindings + data.pop("tool_bindings") + data["built_in_tools"] = body.built_in_tools.model_dump(exclude_none=True) assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data) session.add(assistant) await session.flush() await _sync_bindings(session, assistant.id, resource_ids) - await _sync_tool_bindings(session, assistant.id, assistant.type, tool_ids) + await _sync_tool_bindings( + session, + assistant.id, + assistant.type, + tool_bindings, + body.graph, + ) await session.commit() await session.refresh(assistant) return await _to_out(session, assistant) @@ -488,6 +720,7 @@ async def duplicate_assistant( knowledge_retrieval_config=dict(source.knowledge_retrieval_config or {}), prompt=source.prompt, dynamic_variable_definitions=dict(source.dynamic_variable_definitions or {}), + built_in_tools=dict(source.built_in_tools or {}), analysis_config=dict(source.analysis_config or {}), api_url=source.api_url, api_key=source.api_key, @@ -497,8 +730,14 @@ async def duplicate_assistant( session.add(assistant) await session.flush() await _sync_bindings(session, assistant.id, await _resource_ids(session, source.id)) + source_bindings = await _tool_bindings(session, source.id) + from schemas import ToolBinding await _sync_tool_bindings( - session, assistant.id, assistant.type, await _tool_ids(session, source.id) + session, + assistant.id, + assistant.type, + [ToolBinding(**binding) for binding in source_bindings], + assistant.graph, ) await session.commit() await session.refresh(assistant) @@ -516,19 +755,29 @@ async def update_assistant( raise HTTPException(404, "助手不存在") _validate_workflow(body) await _validate_workflow_references(session, body) - await _validate_system_tool_selection(session, body) + _validate_update_state_config(body) + await _validate_tool_bindings(session, body) + await _validate_prompt_realtime_tool_capabilities(session, body) await _validate_startup_actions(session, body) await _validate_vision_model(session, body) await _validate_analysis_model(session, body) await _validate_knowledge_base(session, body) data = body.model_dump() resource_ids = data.pop("model_resource_ids") - tool_ids = data.pop("tool_ids") + tool_bindings = body.tool_bindings + data.pop("tool_bindings") + data["built_in_tools"] = body.built_in_tools.model_dump(exclude_none=True) data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key) for key, value in data.items(): setattr(assistant, key, value) await _sync_bindings(session, assistant.id, resource_ids) - await _sync_tool_bindings(session, assistant.id, assistant.type, tool_ids) + await _sync_tool_bindings( + session, + assistant.id, + assistant.type, + tool_bindings, + body.graph, + ) await session.commit() await session.refresh(assistant) return await _to_out(session, assistant) diff --git a/backend/routes/mcp_servers.py b/backend/routes/mcp_servers.py index eb9c4ad..ed4757c 100644 --- a/backend/routes/mcp_servers.py +++ b/backend/routes/mcp_servers.py @@ -14,6 +14,7 @@ from schemas import McpServerOut, McpServerUpsert, McpSyncResult, ToolOut from services.auth import require_admin from services.masking import mask_secrets, merge_secrets from services.tool_resources import tool_to_out +from services.system_tools import RESERVED_TOOL_NAMES from services.tools import McpClientError, McpToolClient from sqlalchemy import func, select from sqlalchemy.exc import IntegrityError @@ -27,6 +28,47 @@ router = APIRouter( ) +TEMPLATE_PLACEHOLDER = re.compile(r"{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}") +INPUT_PLACEHOLDER = re.compile(r"{{\s*input__([A-Za-z][A-Za-z0-9_]*)\s*}}") + + +def _validate_template_placeholders(body: McpServerUpsert) -> None: + source = " ".join([body.url, *[str(value) for value in body.headers.values()]]) + invalid = sorted( + name + for name in set(TEMPLATE_PLACEHOLDER.findall(source)) + if not name.startswith(("input__", "system__")) + ) + if invalid: + raise HTTPException( + 400, + "MCP 连接不能直接引用 Assistant 变量；" + "请改为 {{input__name}}: " + ", ".join(invalid), + ) + + +def _template_inputs(server: McpServer) -> list[dict]: + """Expose declared MCP connection placeholders to every synced tool.""" + + config = server.config or {} + source = " ".join( + [ + server.url, + *[str(value) for value in (config.get("headers") or {}).values()], + ] + ) + return [ + { + "name": name, + "type": "string", + "location": "body", + "description": "MCP 连接模板输入", + "required": True, + } + for name in sorted(set(INPUT_PLACEHOLDER.findall(source))) + ] + + def _runtime_server(server: McpServer) -> RuntimeMcpServer: config = server.config or {} secrets = server.secrets or {} @@ -79,6 +121,7 @@ def _apply_body( server: McpServer, body: McpServerUpsert, ) -> None: + _validate_template_placeholders(body) server.name = body.name.strip() server.description = body.description.strip() server.transport = body.transport @@ -109,7 +152,7 @@ async def _available_function_name( base = _function_slug(f"mcp_{server.name}_{remote_name}") candidate = base suffix = 2 - while ( + while candidate in RESERVED_TOOL_NAMES or ( await session.execute( select(Tool.id).where(Tool.function_name == candidate).limit(1) ) @@ -231,22 +274,19 @@ async def sync_mcp_tools( previous_config = (tool.definition or {}).get("config") or {} tool.description = str(remote.get("description") or tool.description) tool.definition = { - "schema_version": 1, + "schema_version": 2, "type": "mcp", "config": { "remote_tool_name": remote_name, "input_schema": remote.get("input_schema") or {}, "schema_hash": remote.get("schema_hash") or "", + "template_inputs": _template_inputs(server), "allow_interruptions": previous_config.get( "allow_interruptions", True ), "execution_mode": previous_config.get( "execution_mode", "immediate" ), - "dynamic_variable_assignments": previous_config.get( - "dynamic_variable_assignments" - ) - or {}, }, } synchronized.append(tool) diff --git a/backend/routes/model_registry.py b/backend/routes/model_registry.py index 1333930..768eb71 100644 --- a/backend/routes/model_registry.py +++ b/backend/routes/model_registry.py @@ -5,9 +5,11 @@ import uuid from db.models import ( Assistant, AssistantModelBinding, + AssistantToolBinding, InterfaceDefinition, KnowledgeBase, ModelResource, + Tool, ) from db.session import get_session from fastapi import APIRouter, Depends, HTTPException, Query @@ -51,6 +53,7 @@ def _definition_dict(row: InterfaceDefinition) -> dict: "name": row.name, "capability": row.capability, "fields": (row.field_schema or {}).get("fields", []), + "features": (row.field_schema or {}).get("features", {}), } @@ -137,6 +140,78 @@ async def _clear_incompatible_references( ) +async def _ensure_realtime_prompt_tools_remain_supported( + session: AsyncSession, + resource: ModelResource, + definition: InterfaceDefinition, +) -> None: + """Reject provider changes that would silently disable bound Prompt tools.""" + + if resource.capability != "Realtime": + return + features = {} + if definition.capability == "Realtime": + features = ( + ((definition.field_schema or {}).get("features") or {}).get( + "realtimePromptTools" + ) + or {} + ) + supported_external = set(features.get("externalTypes") or []) + supported_built_ins = set(features.get("builtIns") or []) + assistants = ( + await session.execute( + select(Assistant) + .join( + AssistantModelBinding, + AssistantModelBinding.assistant_id == Assistant.id, + ) + .where( + AssistantModelBinding.model_resource_id == resource.id, + AssistantModelBinding.capability == "Realtime", + Assistant.type == "prompt", + Assistant.runtime_mode == "realtime", + ) + ) + ).scalars().all() + aliases = { + "end_conversation": "endConversation", + "update_state": "updateState", + "skip_turn": "skipTurn", + "request_human_handoff": "requestHumanHandoff", + } + for assistant in assistants: + requested_built_ins = { + aliases.get(str(name), str(name)) + for name, config in (assistant.built_in_tools or {}).items() + if config is not None + } + if requested_built_ins - supported_built_ins: + raise HTTPException( + 409, + f"模型资源被 Prompt Realtime 助手 {assistant.name} 使用，" + "新接口不支持其内建工具", + ) + tool_types = set( + ( + await session.execute( + select(Tool.type) + .join( + AssistantToolBinding, + AssistantToolBinding.tool_id == Tool.id, + ) + .where(AssistantToolBinding.assistant_id == assistant.id) + ) + ).scalars().all() + ) + if tool_types - supported_external: + raise HTTPException( + 409, + f"模型资源被 Prompt Realtime 助手 {assistant.name} 使用，" + "新接口不支持其外部工具", + ) + + @router.get("/interface-definitions", response_model=list[InterfaceDefinitionOut]) async def list_interface_definitions( capability: str | None = Query(default=None), @@ -252,6 +327,7 @@ async def update_model_resource( if not row: raise HTTPException(404, "模型资源不存在") definition, secrets = await _validate(session, body, row.secrets or {}) + await _ensure_realtime_prompt_tools_remain_supported(session, row, definition) await _clear_incompatible_references(session, row, definition.capability) row.name = body.name row.capability = definition.capability diff --git a/backend/routes/tools.py b/backend/routes/tools.py index 97cd4cb..4fce771 100644 --- a/backend/routes/tools.py +++ b/backend/routes/tools.py @@ -1,5 +1,7 @@ """Reusable tool CRUD. Runtime execution is implemented per supported tool type.""" +import json +import re import uuid from db.models import AssistantToolBinding, Tool @@ -9,6 +11,7 @@ from schemas import ToolOut, ToolUpsert from services.auth import require_admin from services.masking import merge_secrets from services.tool_resources import tool_to_out +from services.system_tools import RESERVED_TOOL_NAMES from sqlalchemy import select from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession @@ -20,17 +23,98 @@ router = APIRouter( dependencies=[Depends(require_admin)], ) +TEMPLATE_PLACEHOLDER = re.compile(r"{{\s*([A-Za-z][A-Za-z0-9_]*)\s*}}") + + +def _normalize_http_template_inputs(definition: dict) -> None: + """Keep Assistant-specific values out of a reusable HTTP resource.""" + + if definition.get("type") != "http": + return + config = definition.get("config") or {} + source = json.dumps( + { + "url": config.get("url"), + "headers": config.get("headers") or {}, + "body": config.get("body") or {}, + }, + ensure_ascii=False, + ) + placeholders = set(TEMPLATE_PLACEHOLDER.findall(source)) + restricted_source = json.dumps( + {"url": config.get("url"), "body": config.get("body") or {}}, + ensure_ascii=False, + ) + misplaced_secrets = sorted( + name + for name in set(TEMPLATE_PLACEHOLDER.findall(restricted_source)) + if name.startswith("secret__") + ) + if misplaced_secrets: + raise HTTPException( + 400, + "工具密钥变量只能用于 HTTP Header: " + + ", ".join(misplaced_secrets), + ) + invalid = sorted( + name + for name in placeholders + if not name.startswith(("input__", "system__", "secret__")) + ) + if invalid: + raise HTTPException( + 400, + "HTTP 资源不能直接引用 Assistant 变量；" + "请改为 {{input__name}}: " + ", ".join(invalid), + ) + names = sorted( + name.removeprefix("input__") + for name in placeholders + if name.startswith("input__") + ) + existing = { + str(item.get("name")): item + for item in config.get("template_inputs") or [] + if item.get("name") + } + config["template_inputs"] = [ + existing.get(name) + or { + "name": name, + "type": "string", + "location": "body", + "description": "HTTP 资源模板输入", + "required": True, + } + for name in names + ] + def _payload( body: ToolUpsert, stored_secrets: dict | None = None, ) -> dict: definition = body.definition.model_dump() + if body.function_name in RESERVED_TOOL_NAMES: + raise HTTPException(400, "工具函数名由平台内建工具保留") + _normalize_http_template_inputs(definition) secrets = ( merge_secrets(body.secrets, stored_secrets or {}) if definition["type"] == "http" else {} ) + secret_variables = (secrets.get("dynamic_variables") or {}) if secrets else {} + invalid_secret_names = sorted( + str(name) + for name in secret_variables + if not str(name).startswith("secret__") + ) + if invalid_secret_names: + raise HTTPException( + 400, + "工具密钥变量必须以 secret__ 开头: " + + ", ".join(invalid_secret_names), + ) return { "name": body.name.strip(), "function_name": body.function_name, @@ -68,7 +152,11 @@ async def _commit(session: AsyncSession, tool: Tool) -> ToolOut: @router.get("", response_model=list[ToolOut]) async def list_tools(session: AsyncSession = Depends(get_session)): rows = ( - await session.execute(select(Tool).order_by(Tool.updated_at.desc())) + await session.execute( + select(Tool) + .where(Tool.type.in_(("http", "mcp", "client"))) + .order_by(Tool.updated_at.desc()) + ) ).scalars().all() return [tool_to_out(tool) for tool in rows] @@ -87,6 +175,8 @@ async def get_tool(tool_id: str, session: AsyncSession = Depends(get_session)): tool = await session.get(Tool, tool_id) if not tool: raise HTTPException(404, "工具不存在") + if tool.type == "system": + raise HTTPException(404, "工具不存在") return tool_to_out(tool) @@ -99,6 +189,8 @@ async def update_tool( tool = await session.get(Tool, tool_id) if not tool: raise HTTPException(404, "工具不存在") + if tool.type == "system": + raise HTTPException(404, "工具不存在") if tool.type == "mcp": if body.definition.type != "mcp": raise HTTPException(400, "同步生成的 MCP 工具不能修改类型") @@ -117,6 +209,8 @@ async def duplicate_tool(tool_id: str, session: AsyncSession = Depends(get_sessi source = await session.get(Tool, tool_id) if not source: raise HTTPException(404, "工具不存在") + if source.type == "system": + raise HTTPException(404, "工具不存在") if source.type == "mcp": raise HTTPException(400, "MCP 工具由 Server 同步维护，不能单独复制") function_name = _duplicate_function_name(source.function_name) @@ -148,6 +242,8 @@ async def delete_tool(tool_id: str, session: AsyncSession = Depends(get_session) tool = await session.get(Tool, tool_id) if not tool: raise HTTPException(404, "工具不存在") + if tool.type == "system": + raise HTTPException(404, "工具不存在") in_use = ( await session.execute( select(AssistantToolBinding.assistant_id) diff --git a/backend/routes/voice_webrtc.py b/backend/routes/voice_webrtc.py index b9a5b3c..a21f9c0 100644 --- a/backend/routes/voice_webrtc.py +++ b/backend/routes/voice_webrtc.py @@ -90,6 +90,7 @@ async def http_offer(request: Request, payload: dict = Body(...)): **payload, "assistant_id": request.headers.get("x-pipecat-assistant-id"), "dynamic_variables": header_variables, + "client_tool_names": request_data.get("client_tool_names") or [], **request_data, } answer = await _handle_offer_payload(offer_payload, _http_peers) @@ -225,6 +226,7 @@ async def _handle_offer_payload(payload, peers): vision_enabled=vision_enabled, assistant_id=offer.assistant_id, channel="webrtc", + client_tool_names=set(offer.client_tool_names), ), ) diff --git a/backend/schemas.py b/backend/schemas.py index caee996..e7c05e4 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -9,7 +9,7 @@ from __future__ import annotations from datetime import UTC, datetime import re -from typing import Annotated, Any, Literal, Union +from typing import Annotated, Any, ClassVar, Literal, Union from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic.alias_generators import to_camel @@ -20,7 +20,7 @@ ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent"] AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"] TurnEndStrategy = Literal["silence", "smart_turn"] KnowledgeRetrievalMode = Literal["automatic", "on_demand"] -ToolType = Literal["system", "http", "mcp", "client"] +ToolType = Literal["http", "mcp", "client"] ToolStatus = Literal["active", "archived", "draft"] McpServerStatus = Literal["active", "archived", "draft"] McpTransport = Literal["streamable_http", "sse"] @@ -31,12 +31,7 @@ ClientToolResponseWaitMode = Literal["timeout", "session"] DynamicVariableType = Literal["string", "number", "boolean"] PromptEntryMode = Literal["wait_user", "generate"] PromptOpeningMode = Literal["interruptible", "playback", "confirmation"] -SystemToolKind = Literal[ - "end_conversation", - "update_state", - "skip_turn", - "request_human_handoff", -] +PrimitiveValue = str | int | float | bool # 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵 EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"} @@ -140,11 +135,55 @@ ALLOWED_FIELDS: dict[str, set[str]] = { } +class ModelValueSource(CamelModel): + source: Literal["model"] = "model" + + +class VariableValueSource(CamelModel): + source: Literal["variable"] = "variable" + variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$") + + +class LiteralValueSource(CamelModel): + source: Literal["literal"] = "literal" + value: Any = None + + +class TemplateValueSource(CamelModel): + source: Literal["template"] = "template" + value: Any + + +ValueSource = Annotated[ + Union[ + ModelValueSource, + VariableValueSource, + LiteralValueSource, + TemplateValueSource, + ], + Field(discriminator="source"), +] + + +NonModelValueSource = Annotated[ + Union[VariableValueSource, LiteralValueSource, TemplateValueSource], + Field(discriminator="source"), +] + + +class ToolBinding(CamelModel): + """One assistant/stage-specific use of a reusable external tool.""" + + tool_id: str = Field(min_length=1, max_length=40) + argument_bindings: dict[str, ValueSource] = Field(default_factory=dict) + template_bindings: dict[str, NonModelValueSource] = Field(default_factory=dict) + result_assignments: dict[str, str] = Field(default_factory=dict) + + class StartupAction(CamelModel): id: str = Field(min_length=1, max_length=64, pattern=r"^[A-Za-z][A-Za-z0-9_-]*$") phase: Literal["preflight", "opening"] = "opening" - tool_id: str = Field(min_length=1, max_length=40) - arguments: dict[str, Any] = Field(default_factory=dict) + tool_binding: ToolBinding required: bool = True @@ -190,6 +229,11 @@ class StartupConfig(CamelModel): # ---------- 助手(单表 STI:瘦类型真列 + workflow 图 JSON 列) ---------- class AssistantUpsert(CamelModel): + # AssistantOut inherits this DTO for a stable public shape, but persisted + # historical rows must remain readable. Only request models enforce + # runtime combinations that the current editor/runtime can create. + enforce_request_constraints: ClassVar[bool] = True + name: str type: AssistantType = "prompt" runtime_mode: RuntimeMode = "pipeline" @@ -206,7 +250,8 @@ class AssistantUpsert(CamelModel): knowledge_retrieval_config: KnowledgeRetrievalConfig = Field( default_factory=KnowledgeRetrievalConfig ) - tool_ids: list[str] = Field(default_factory=list) + tool_bindings: list[ToolBinding] = Field(default_factory=list) + built_in_tools: "BuiltInTools" = Field(default_factory=lambda: BuiltInTools()) # 瘦类型专属(真列);按 type 取用,无关字段写入时清零 prompt: str = "" @@ -244,17 +289,24 @@ class AssistantUpsert(CamelModel): if self.type == "workflow": self.greeting = "" if self.type not in {"prompt", "workflow"}: - self.tool_ids = [] + self.tool_bindings = [] + self.built_in_tools = BuiltInTools() self.dynamic_variable_definitions = {} if self.type != "prompt": self.startup = StartupConfig() - if self.runtime_mode == "realtime" and ( - self.startup.actions - or self.startup.opening_message is not None - or self.startup.opening_mode != "interruptible" - or self.startup.entry_mode != "wait_user" - ): - raise ValueError("Prompt Realtime 模式暂不支持启动阶段") + if self.type == "workflow": + # Workflow usage configuration lives inside graph v4. Top-level + # bindings remain a Prompt-only public API. + self.tool_bindings = [] + self.built_in_tools = BuiltInTools() + if self.enforce_request_constraints: + if self.runtime_mode == "realtime" and ( + self.startup.actions + or self.startup.opening_message is not None + or self.startup.opening_mode != "interruptible" + or self.startup.entry_mode != "wait_user" + ): + raise ValueError("Prompt Realtime 模式暂不支持启动阶段") # 外部托管大脑只能 cascade,拦住不兼容的 realtime if self.runtime_mode == "realtime" and self.type not in REALTIME_CAPABLE_TYPES: raise ValueError(f"类型 {self.type} 不支持 realtime 运行模式") @@ -262,6 +314,8 @@ class AssistantUpsert(CamelModel): class AssistantOut(AssistantUpsert): + enforce_request_constraints: ClassVar[bool] = False + id: str updated_at: str | None = None @@ -289,6 +343,94 @@ class DynamicVariableDefinition(CamelModel): return self +# ---------- 平台内建工具 ---------- +class ConstantStateExpression(CamelModel): + type: Literal["constant"] = "constant" + value: PrimitiveValue + + +class VariableStateExpression(CamelModel): + type: Literal["variable"] = "variable" + variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$") + + +class LlmStateExpression(CamelModel): + type: Literal["llm"] = "llm" + parameter_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$") + value_type: DynamicVariableType = "string" + description: str = Field(min_length=1, max_length=500) + + +class OperationStateExpression(CamelModel): + type: Literal[ + "and", + "or", + "eq", + "neq", + "gt", + "gte", + "lt", + "lte", + "add", + "subtract", + "multiply", + "divide", + ] + operands: list["StateExpression"] = Field(min_length=2, max_length=10) + + +class ConditionalStateExpression(CamelModel): + type: Literal["if"] = "if" + condition: "StateExpression" + then: "StateExpression" + otherwise: "StateExpression" + + +StateExpression = Annotated[ + Union[ + ConstantStateExpression, + VariableStateExpression, + LlmStateExpression, + OperationStateExpression, + ConditionalStateExpression, + ], + Field(discriminator="type"), +] + + +class StateUpdateRule(CamelModel): + variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$") + expression: StateExpression + + +class BuiltInToolBase(CamelModel): + description: str = Field(default="", max_length=2048) + + +class EndConversationTool(BuiltInToolBase): + message_type: Literal["none", "custom"] = "none" + custom_message: str = Field(default="", max_length=2000) + capture_reason: bool = True + allow_end_interruption: bool = False + + +class UpdateStateTool(BuiltInToolBase): + updates: list[StateUpdateRule] = Field(default_factory=list, max_length=10) + + +class BuiltInTools(CamelModel): + end_conversation: EndConversationTool | None = None + update_state: UpdateStateTool | None = None + skip_turn: BuiltInToolBase | None = None + request_human_handoff: BuiltInToolBase | None = None + + +OperationStateExpression.model_rebuild() +ConditionalStateExpression.model_rebuild() +AssistantUpsert.model_rebuild() +AssistantOut.model_rebuild() + + # ---------- 可复用工具 ---------- class ToolParameter(CamelModel): name: str @@ -298,14 +440,6 @@ class ToolParameter(CamelModel): required: bool = True -class SystemToolConfig(CamelModel): - kind: SystemToolKind = "end_conversation" - message_type: Literal["none", "custom"] = "none" - custom_message: str = "" - capture_reason: bool = True - allow_end_interruption: bool = False - - class HttpToolConfig(CamelModel): allow_interruptions: bool = True execution_mode: ToolExecutionMode = "immediate" @@ -314,8 +448,8 @@ class HttpToolConfig(CamelModel): timeout_seconds: int = Field(default=15, ge=1, le=120) headers: dict[str, str] = Field(default_factory=dict) parameters: list[ToolParameter] = Field(default_factory=list) + template_inputs: list[ToolParameter] = Field(default_factory=list) body: dict[str, Any] = Field(default_factory=dict) - dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict) @field_validator("url") @classmethod @@ -325,14 +459,8 @@ class HttpToolConfig(CamelModel): return value -class SystemToolDefinition(CamelModel): - schema_version: int = 1 - type: Literal["system"] = "system" - config: SystemToolConfig = Field(default_factory=SystemToolConfig) - - class HttpToolDefinition(CamelModel): - schema_version: int = 1 + schema_version: int = 2 type: Literal["http"] = "http" config: HttpToolConfig @@ -344,17 +472,10 @@ class ClientToolConfig(CamelModel): response_wait_mode: ClientToolResponseWaitMode = "timeout" parameters: list[ToolParameter] = Field(default_factory=list) timeout_seconds: int = Field(default=3, ge=1, le=30) - dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict) - - @model_validator(mode="after") - def validate_result_assignments(self): - if not self.wait_for_response and self.dynamic_variable_assignments: - raise ValueError("Client Tool 不等待响应时不能配置结果变量赋值") - return self class ClientToolDefinition(CamelModel): - schema_version: int = 1 + schema_version: int = 2 type: Literal["client"] = "client" config: ClientToolConfig = Field(default_factory=ClientToolConfig) @@ -365,18 +486,17 @@ class McpToolConfig(CamelModel): remote_tool_name: str = Field(min_length=1, max_length=255) input_schema: dict[str, Any] = Field(default_factory=dict) schema_hash: str = "" - dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict) + template_inputs: list[ToolParameter] = Field(default_factory=list) class McpToolDefinition(CamelModel): - schema_version: int = 1 + schema_version: int = 2 type: Literal["mcp"] = "mcp" config: McpToolConfig ToolDefinition = Annotated[ Union[ - SystemToolDefinition, HttpToolDefinition, ClientToolDefinition, McpToolDefinition, diff --git a/backend/services/action_runtime.py b/backend/services/action_runtime.py index 872a1bc..7db80ce 100644 --- a/backend/services/action_runtime.py +++ b/backend/services/action_runtime.py @@ -10,7 +10,7 @@ from time import monotonic from typing import Any from uuid import uuid4 -from models import RuntimeTool +from models import RuntimeTool, RuntimeToolBinding from services.tool_executor import ToolExecutionError, ToolExecutor @@ -101,6 +101,7 @@ class ActionRunner: tool: RuntimeTool | None, arguments: dict[str, Any] | None = None, *, + binding: RuntimeToolBinding | None = None, result_assignments: dict[str, str] | None = None, invocation_id: str | None = None, ) -> ActionOutcome: @@ -120,6 +121,7 @@ class ActionRunner: result = await self._executor.execute( tool, rendered_arguments, + binding=binding, result_assignments=result_assignments, ) if result.get("status") != "ok": diff --git a/backend/services/action_stage.py b/backend/services/action_stage.py index 1de949f..9aee6f7 100644 --- a/backend/services/action_stage.py +++ b/backend/services/action_stage.py @@ -6,7 +6,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass, field from typing import Any, Literal -from models import RuntimeTool +from models import RuntimeTool, RuntimeToolBinding from services.action_runtime import ActionOutcome, ActionRunner, ActionStatus @@ -22,6 +22,7 @@ class StageAction: id: str tool: RuntimeTool | None arguments: dict[str, Any] = field(default_factory=dict) + binding: RuntimeToolBinding | None = None result_assignments: dict[str, str] | None = None required: bool = True invocation_id: str | None = None @@ -75,6 +76,7 @@ class ActionStageRunner: outcome = await self._actions.execute( action.tool, action.arguments, + binding=action.binding, result_assignments=action.result_assignments, invocation_id=action.invocation_id, ) diff --git a/backend/services/brains/prompt_brain.py b/backend/services/brains/prompt_brain.py index a84a64d..35c6c1b 100644 --- a/backend/services/brains/prompt_brain.py +++ b/backend/services/brains/prompt_brain.py @@ -9,7 +9,7 @@ from typing import Any from uuid import uuid4 from loguru import logger -from models import AssistantConfig +from models import AssistantConfig, RuntimeToolBinding from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.frames.frames import ( LLMRunFrame, @@ -28,8 +28,10 @@ from services.brains.base import ( BaseBrain, BrainRuntime, BrainSpec, + RealtimeBrainRuntime, SessionVariableUpdate, ) +from services.brains.prompt_realtime_tools import PromptRealtimeToolController from services.action_runtime import ( ActionOutcome, ActionInvocationCancelled, @@ -46,7 +48,8 @@ from services.message_stage import ( confirmation_context_message, ) from services.runtime_variables import DynamicVariableError, DynamicVariableStore -from services.system_tools import state_update_properties, system_tool_kind +from services.system_tools import runtime_system_tools, state_update_properties, system_tool_kind +from services.state_expressions import evaluate_update_rules from services.tool_executor import ToolExecutionError, ToolExecutor from services.tool_policy import policy_for_tool @@ -70,6 +73,7 @@ class PromptBrain(BaseBrain): self._message_stages = MessageStageRunner() self._tool_by_id = {tool.id: tool for tool in cfg.tools} self._runtime: BrainRuntime | None = None + self._realtime_controller: PromptRealtimeToolController | None = None self._output: FixedSpeechOutput | None = None self._waiting_for_generated_end_speech = False self._preflight_finished = False @@ -123,18 +127,33 @@ class PromptBrain(BaseBrain): self._startup_failed = False self._greeting_pending = False self._entry_dispatched = False - llm_tool_ids = ( - set(cfg.llm_tool_ids) if cfg.llm_tool_ids is not None else None - ) schemas: list[FunctionSchema] = [] registered_names: set[str] = set() - for tool in cfg.tools: - if llm_tool_ids is not None and tool.id not in llm_tool_ids: + binding_by_id = {binding.tool_id: binding for binding in cfg.tool_bindings} + selected_tools = [ + self._tool_by_id[binding.tool_id] + for binding in cfg.tool_bindings + if binding.tool_id in self._tool_by_id + ] + selected_tools.extend(runtime_system_tools(cfg.built_in_tools)) + for tool in selected_tools: + if ( + tool.type == "client" + and runtime.client_tools is not None + and getattr(runtime.client_tools, "supports", lambda _name: True)( + tool.function_name + ) is False + ): + logger.warning(f"跳过客户端未上报的工具: {tool.function_name}") continue if tool.type == "system": schema, handler = self._make_system_tool(tool, runtime) elif tool.type in {"http", "mcp", "client"}: - schema, handler = self._make_remote_tool(tool, runtime) + schema, handler = self._make_remote_tool( + tool, + runtime, + binding_by_id[tool.id], + ) else: continue if schema.name in registered_names: @@ -152,6 +171,45 @@ class PromptBrain(BaseBrain): ) runtime.set_tools(schemas) + async def setup_realtime( + self, + cfg: AssistantConfig, + runtime: RealtimeBrainRuntime, + ) -> None: + """Install Prompt-owned tools into a supported realtime provider.""" + + self._cfg = cfg + self._store = DynamicVariableStore.from_config(cfg) + self._tool_by_id = {tool.id: tool for tool in cfg.tools} + self._realtime_controller = PromptRealtimeToolController( + cfg=cfg, + store=self._store, + runtime=runtime, + ) + runtime.realtime.set_speech_started_handler( + self.on_realtime_user_speech_started + ) + set_response_done_handler = getattr( + runtime.realtime, + "set_response_done_handler", + None, + ) + if callable(set_response_done_handler): + set_response_done_handler(self.on_realtime_response_done) + await self._realtime_controller.start() + + async def on_realtime_user_speech_started(self) -> None: + if self._realtime_controller is not None: + await self._realtime_controller.on_user_speech_started() + + async def on_realtime_response_done( + self, + content: str, + interrupted: bool, + ) -> None: + if self._realtime_controller is not None: + await self._realtime_controller.on_response_done(content, interrupted) + async def run_preflight(self) -> None: if self._preflight_finished: return @@ -367,20 +425,21 @@ class PromptBrain(BaseBrain): ] def _opening_actions_stage_spec(self) -> ActionStageSpec: - actions = tuple( - StageAction( - id=str(action.get("id") or "startup_action"), - tool=self._tool_by_id.get( - str(action.get("tool_id") or action.get("toolId") or "") - ), - arguments=action.get("arguments") or {}, - required=bool(action.get("required", True)), - invocation_id=self._actions.new_invocation_id(), + actions = [] + for action in self._startup_actions("opening"): + raw_binding = action.get("tool_binding") or action.get("toolBinding") or {} + binding = RuntimeToolBinding(**raw_binding) + actions.append( + StageAction( + id=str(action.get("id") or "startup_action"), + tool=self._tool_by_id.get(binding.tool_id), + binding=binding, + required=bool(action.get("required", True)), + invocation_id=self._actions.new_invocation_id(), + ) ) - for action in self._startup_actions("opening") - ) return ActionStageSpec( - actions=actions, + actions=tuple(actions), input_policy="block", ) @@ -441,7 +500,9 @@ class PromptBrain(BaseBrain): async def _run_startup_actions(self, phase: str) -> bool: for action in self._startup_actions(phase): action_id = str(action.get("id") or "startup_action") - tool_id = str(action.get("tool_id") or action.get("toolId") or "") + raw_binding = action.get("tool_binding") or action.get("toolBinding") or {} + binding = RuntimeToolBinding(**raw_binding) + tool_id = binding.tool_id tool = self._tool_by_id.get(tool_id) invocation_id = self._actions.new_invocation_id() logger.info( @@ -449,7 +510,7 @@ class PromptBrain(BaseBrain): ) outcome = await self._actions.execute( tool, - action.get("arguments") or {}, + binding=binding, invocation_id=invocation_id, ) if outcome.updated_variables: @@ -504,6 +565,8 @@ class PromptBrain(BaseBrain): async def on_assistant_text_start(self, _turn_id: str) -> None: if self._runtime is not None: self._runtime.call_end.begin_response() + if self._realtime_controller is not None: + self._realtime_controller.on_assistant_text_start() async def on_assistant_text_end( self, @@ -520,14 +583,18 @@ class PromptBrain(BaseBrain): await self._runtime.call_end.finish_after_current_speech( has_text=bool(content.strip()) and not interrupted ) + if self._realtime_controller is not None: + await self._realtime_controller.on_assistant_text_end( + content, + interrupted, + ) def _refresh_prompt(self) -> None: if self._dynamic_enabled and self._runtime is not None: self._runtime.set_system_prompt(self._store.render(self._cfg.prompt)) - def _make_remote_tool(self, tool, runtime: BrainRuntime): - properties, required = self._tools.schema_parts(tool) - self._tools.register_secrets(tool) + def _make_remote_tool(self, tool, runtime: BrainRuntime, binding): + properties, required = self._tools.schema_parts(tool, binding) policy = policy_for_tool(tool) async def return_result(params: FunctionCallParams, result: dict) -> None: @@ -551,7 +618,11 @@ class PromptBrain(BaseBrain): toolType=tool.type, ) try: - result = await self._tools.execute(tool, dict(params.arguments or {})) + result = await self._tools.execute( + tool, + dict(params.arguments or {}), + binding=binding, + ) if result["updated_variables"]: self._refresh_prompt() await self._emit_trace( @@ -690,7 +761,7 @@ class PromptBrain(BaseBrain): ) return schema, end_call - # ---------- System 工具资源 ---------- + # ---------- 平台内建对话工具 ---------- def _make_system_tool(self, tool, runtime: BrainRuntime): kind = system_tool_kind(tool.definition or {}) @@ -707,7 +778,7 @@ class PromptBrain(BaseBrain): else: raise ValueError(f"未知系统工具: {kind}") - # System tools are platform-owned deterministic behavior. Keep their + # Built-ins are platform-owned deterministic behavior. Keep their # lifecycle and state semantics identical across production and tests; # only external HTTP, MCP, and client tools use an injected executor. return schema, handler @@ -715,13 +786,22 @@ class PromptBrain(BaseBrain): def _make_update_state_tool(self, tool, runtime: BrainRuntime): """更新已声明的动态变量(会话状态),并让模型继续当前回答。""" - writable = state_update_properties(self._cfg.dynamic_variable_definitions) + config = (tool.definition or {}).get("config") or {} + rules = list(config.get("updates") or []) + writable = state_update_properties( + self._cfg.dynamic_variable_definitions, + rules=rules, + ) async def update_state(params: FunctionCallParams) -> None: - state = dict(params.arguments or {}) try: + state = evaluate_update_rules( + rules, + dict(self._store.values), + dict(params.arguments or {}), + ) changed = self._store.assign_declared_many(state) - except DynamicVariableError as exc: + except (DynamicVariableError, ValueError) as exc: await params.result_callback( {"status": "error", "message": f"状态更新失败: {exc}"} ) diff --git a/backend/services/brains/prompt_realtime_tools.py b/backend/services/brains/prompt_realtime_tools.py new file mode 100644 index 0000000..9d78e51 --- /dev/null +++ b/backend/services/brains/prompt_realtime_tools.py @@ -0,0 +1,530 @@ +"""Prompt-owned tools for Qwen speech-to-speech sessions.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from time import monotonic +from typing import Any +from uuid import uuid4 + +from loguru import logger +from models import AssistantConfig, RuntimeTool, RuntimeToolBinding +from pipecat.frames.frames import OutputTransportMessageUrgentFrame +from pipecat.utils.time import time_now_iso8601 + +from services.brains.base import RealtimeBrainRuntime +from services.pipecat.call_lifecycle import playback_marker_for +from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult +from services.runtime_variables import DynamicVariableError, DynamicVariableStore +from services.state_expressions import evaluate_update_rules +from services.system_tools import ( + RESERVED_TOOL_NAMES, + runtime_system_tools, + state_update_properties, + system_tool_kind, +) +from services.tool_executor import ToolExecutor +from services.tool_policy import policy_for_tool + + +ToolHandler = Callable[[dict[str, Any]], Awaitable[RealtimeToolResult]] +SUPPORTED_BUILT_INS = frozenset( + {"end_conversation", "update_state", "request_human_handoff"} +) +MAX_PUBLIC_TOOL_ERROR = 512 + + +class PromptRealtimeToolController: + """Install and execute Prompt tools without modifying Pipecat itself.""" + + def __init__( + self, + *, + cfg: AssistantConfig, + store: DynamicVariableStore, + runtime: RealtimeBrainRuntime, + ) -> None: + self._cfg = cfg + self._store = store + self._runtime = runtime + self._executor = ToolExecutor(store, client_tools=runtime.client_tools) + self._tools_by_id = {tool.id: tool for tool in cfg.tools} + self._handlers: dict[str, ToolHandler] = {} + self._input_locks = 0 + self._handoff_requested = False + self._waiting_generated_end = False + + async def start(self) -> None: + if self._cfg.realtime_interface_type != "qwen-audio-realtime": + if self._cfg.tool_bindings or runtime_system_tools( + self._cfg.built_in_tools + ): + raise ValueError( + "当前 Realtime Provider 不支持 Prompt 工具调用" + ) + return + tools = await self._build_tools() + self._runtime.realtime.set_tool_dispatcher(self.dispatch_tool) + await self._runtime.realtime.update_session( + self._store.render(self._cfg.prompt), + tools, + ) + + async def dispatch_tool( + self, + name: str, + arguments: dict[str, Any], + _call_id: str, + ) -> RealtimeToolResult: + handler = self._handlers.get(name) + if handler is None: + return RealtimeToolResult( + {"status": "error", "message": f"当前会话没有工具:{name}"} + ) + return await handler(arguments) + + async def on_user_speech_started(self) -> None: + """Cancel only an explicitly interruptible pending ending.""" + + if self._runtime.call_end.cancel_pending_end(): + self._waiting_generated_end = False + self._set_input_enabled(True, force=True) + + def on_assistant_text_start(self) -> None: + """Reset transport speech tracking for the response that just started.""" + + self._runtime.call_end.begin_response() + + async def on_assistant_text_end( + self, + content: str, + interrupted: bool, + ) -> None: + if not self._waiting_generated_end: + return + self._waiting_generated_end = False + if self._runtime.call_end.ending: + await self._runtime.call_end.finish_after_current_speech( + has_text=bool(content.strip()) and not interrupted + ) + + async def on_response_done(self, content: str, interrupted: bool) -> None: + """Finish a generated ending even when the response has no transcript.""" + + if self._waiting_generated_end: + await self.on_assistant_text_end(content, interrupted) + + async def _build_tools(self) -> list[RealtimeTool]: + result: list[RealtimeTool] = [] + binding_by_id = { + binding.tool_id: binding for binding in self._cfg.tool_bindings + } + + async def add( + tool: RealtimeTool, + handler: ToolHandler, + *, + source_id: str, + ) -> None: + if tool.name in self._handlers: + await self._emit_trace( + "tool_skipped", + toolId=source_id, + functionName=tool.name, + reason="duplicate_function_name", + ) + return + self._handlers[tool.name] = handler + result.append(tool) + + for binding in self._cfg.tool_bindings: + runtime_tool = self._tools_by_id.get(binding.tool_id) + if runtime_tool is None or runtime_tool.type not in { + "http", + "mcp", + "client", + }: + continue + if runtime_tool.function_name in RESERVED_TOOL_NAMES: + raise ValueError( + f"外部工具使用了平台保留函数名:{runtime_tool.function_name}" + ) + if ( + runtime_tool.type == "client" + and ( + self._runtime.client_tools is None + or getattr( + self._runtime.client_tools, + "supports", + lambda _name: True, + )(runtime_tool.function_name) + is False + ) + ): + logger.warning( + f"跳过客户端未上报的 Realtime 工具: " + f"{runtime_tool.function_name}" + ) + await self._emit_trace( + "tool_skipped", + toolId=runtime_tool.id, + functionName=runtime_tool.function_name, + toolType=runtime_tool.type, + reason="client_capability_missing", + ) + continue + built = self._external_tool(runtime_tool, binding_by_id[binding.tool_id]) + await add(*built, source_id=runtime_tool.id) + + for runtime_tool in runtime_system_tools(self._cfg.built_in_tools): + kind = system_tool_kind(runtime_tool.definition or {}) + if kind == "skip_turn": + raise ValueError("Prompt Realtime 暂不支持 skip_turn") + if kind not in SUPPORTED_BUILT_INS: + continue + built = self._system_tool(runtime_tool) + if built is not None: + await add(*built, source_id=runtime_tool.id) + return result + + def _external_tool( + self, + tool: RuntimeTool, + binding: RuntimeToolBinding, + ) -> tuple[RealtimeTool, ToolHandler]: + properties, required = self._executor.schema_parts(tool, binding) + policy = policy_for_tool(tool) + + async def handler(arguments: dict[str, Any]) -> RealtimeToolResult: + invocation_id = f"tool_{uuid4().hex[:20]}" + started_at = monotonic() + await self._emit_trace( + "tool_started", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + ) + if not policy.allow_interruptions: + self._lock_input() + try: + result = await self._executor.execute( + tool, + arguments, + binding=binding, + ) + changed = list(result.get("updated_variables") or []) + if changed: + await self._publish_variables("tool", changed) + await self._refresh_instructions() + await self._emit_trace( + "tool_completed" + if result.get("status") == "ok" + else "tool_failed", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status=str(result.get("status") or "unknown"), + durationMs=max(0, round((monotonic() - started_at) * 1000)), + updatedVariables=changed, + resultKeys=sorted(str(name) for name in result if name != "data"), + ) + follow_up = ( + "continue" if policy.runs_llm_after_result else "none" + ) + return RealtimeToolResult( + result, + follow_up=follow_up, + ) + except Exception as exc: # noqa: BLE001 - sanitize provider errors + message = self._public_tool_error(exc) + await self._emit_trace( + "tool_failed", + invocationId=invocation_id, + toolId=tool.id, + toolName=tool.name, + functionName=tool.function_name, + toolType=tool.type, + status="error", + durationMs=max(0, round((monotonic() - started_at) * 1000)), + error=message, + ) + follow_up = ( + "continue" if policy.runs_llm_after_result else "none" + ) + return RealtimeToolResult( + {"status": "error", "message": message}, + follow_up=follow_up, + ) + finally: + if not policy.allow_interruptions: + self._unlock_input() + + return ( + RealtimeTool( + name=tool.function_name, + description=tool.description or f"调用 {tool.name}", + properties=properties, + required=tuple(required), + ), + handler, + ) + + def _system_tool( + self, + tool: RuntimeTool, + ) -> tuple[RealtimeTool, ToolHandler] | None: + kind = system_tool_kind(tool.definition or {}) + config = (tool.definition or {}).get("config") or {} + if kind == "update_state": + rules = list(config.get("updates") or []) + + async def update_state(arguments: dict[str, Any]) -> RealtimeToolResult: + try: + values = evaluate_update_rules( + rules, + dict(self._store.values), + arguments, + ) + changed = self._store.assign_declared_many(values) + except (DynamicVariableError, ValueError) as exc: + return RealtimeToolResult( + { + "status": "error", + "message": f"状态更新失败:{self._public_error(exc)}", + } + ) + if changed: + await self._publish_variables("update_state", changed) + await self._refresh_instructions() + return RealtimeToolResult( + { + "status": "success", + "changed": changed, + "variables": self._store.public_values(), + } + ) + + return ( + RealtimeTool( + name=tool.function_name, + description=tool.description or "更新已授权的动态变量。", + properties=state_update_properties( + self._cfg.dynamic_variable_definitions, + rules=rules, + ), + ), + update_state, + ) + + if kind == "request_human_handoff": + async def request_handoff( + arguments: dict[str, Any], + ) -> RealtimeToolResult: + reason = str(arguments.get("reason") or "human_handoff") + if not self._handoff_requested: + self._handoff_requested = True + self._store.values["system__handoff_status"] = "requested" + await self._runtime.queue_frame( + OutputTransportMessageUrgentFrame( + message={ + "type": "handoff-requested", + "source": "prompt-system-tool", + "reason": reason, + "message": "用户请求转接人工服务。", + } + ) + ) + await self._publish_variables( + "human_handoff", + ["system__handoff_status"], + ) + await self._refresh_instructions() + return RealtimeToolResult( + { + "status": "requested", + "action": "human_handoff_requested", + "message": "人工接管请求已提交，请告知用户正在等待人工响应。", + } + ) + + return ( + RealtimeTool( + name=tool.function_name, + description=tool.description or "提交人工接管请求。", + properties={ + "reason": {"type": "string", "description": "转接原因"} + }, + ), + request_handoff, + ) + + if kind == "end_conversation": + capture_reason = bool(config.get("capture_reason", True)) + allow_end_interruption = bool( + config.get("allow_end_interruption", False) + ) + message_type = str(config.get("message_type") or "none") + custom_message = str(config.get("custom_message") or "").strip() + + async def end_conversation( + arguments: dict[str, Any], + ) -> RealtimeToolResult: + if self._runtime.call_end.ending: + return RealtimeToolResult( + {"status": "success", "action": "ending_call"}, + follow_up="suppress", + ) + reason = str( + arguments.get("reason") or "end_conversation" + ).strip() + self._runtime.call_end.begin( + reason, + interruptible=allow_end_interruption, + ) + if not allow_end_interruption: + self._set_input_enabled(False, force=True) + uses_custom_message = ( + message_type == "custom" and bool(custom_message) + ) + if uses_custom_message: + return RealtimeToolResult( + {"status": "success", "action": "ending_call"}, + follow_up="suppress", + after_output=lambda: self._play_end_message(custom_message), + ) + self._waiting_generated_end = True + return RealtimeToolResult( + { + "status": "success", + "action": "ending_call", + "message": "请用一句简短、礼貌的结束语结束对话。", + } + ) + + properties = ( + { + "reason": { + "type": "string", + "description": "结束本次通话的简短原因。", + } + } + if capture_reason + else {} + ) + return ( + RealtimeTool( + name=tool.function_name, + description=tool.description or "礼貌结束本次对话。", + properties=properties, + required=("reason",) if capture_reason else (), + ), + end_conversation, + ) + return None + + async def _play_end_message(self, content: str) -> None: + if not self._runtime.call_end.ending: + return + turn_id = uuid4().hex + timestamp = time_now_iso8601() + for message in ( + {"type": "assistant-text-start", "turn_id": turn_id, "timestamp": timestamp}, + {"type": "assistant-text-delta", "turn_id": turn_id, "delta": content}, + { + "type": "assistant-text-end", + "turn_id": turn_id, + "content": content, + "interrupted": False, + }, + ): + await self._runtime.queue_frame( + OutputTransportMessageUrgentFrame(message=message) + ) + track_speech = getattr(self._runtime.call_end, "track_speech", None) + playback_completion = track_speech() if callable(track_speech) else None + provider_completion = await self._runtime.realtime.speak_fixed( + content, + suppress_transcript=True, + ) + marker = playback_marker_for(playback_completion) + if marker is not None: + if provider_completion is not None: + await provider_completion + await self._runtime.queue_frame(marker) + marker.completion.mark_queued() + await self._runtime.call_end.arm_after_tracked_speech() + await playback_completion + return + self._runtime.call_end.arm_after_speech() + + async def _refresh_instructions(self) -> None: + await self._runtime.realtime.update_instructions( + self._store.render(self._cfg.prompt) + ) + + async def _publish_variables(self, reason: str, changed: list[str]) -> None: + await self._runtime.queue_frame( + OutputTransportMessageUrgentFrame( + message={ + "type": "session-variables", + "reason": reason, + "variables": self._store.public_values(), + "changed": changed, + } + ) + ) + + async def _emit_trace(self, event: str, **details: Any) -> None: + try: + await self._runtime.queue_frame( + OutputTransportMessageUrgentFrame( + message={ + "type": "workflow-event", + "eventId": f"wfe_{uuid4().hex[:20]}", + "event": event, + "timestamp": time_now_iso8601(), + "sessionId": self._runtime.session_id, + "source": "prompt-realtime", + **details, + } + ) + ) + except Exception as exc: # noqa: BLE001 - trace must not break tools + logger.warning(f"发送 Prompt Realtime 工具轨迹失败:{exc}") + + def _lock_input(self) -> None: + self._input_locks += 1 + self._set_input_enabled(False) + + def _unlock_input(self) -> None: + self._input_locks = max(0, self._input_locks - 1) + if self._input_locks == 0 and not self._runtime.call_end.ending: + self._set_input_enabled(True) + + def _set_input_enabled(self, enabled: bool, *, force: bool = False) -> None: + if self._runtime.set_input_enabled is None: + return + if force or enabled is False or self._input_locks == 0: + self._runtime.set_input_enabled(enabled) + + @staticmethod + def _public_error(exc: Exception) -> str: + message = str(exc).strip() or type(exc).__name__ + return message[:MAX_PUBLIC_TOOL_ERROR] + + @staticmethod + def _public_tool_error(exc: Exception) -> str: + """Return actionable errors without exposing URLs, payloads, or secrets.""" + + message = str(exc).strip() + if "超时" in message: + return "工具调用超时" + if message.startswith("客户端未注册工具"): + return message[:MAX_PUBLIC_TOOL_ERROR] + if any(token in message for token in ("参数", "变量", "绑定")): + return message[:MAX_PUBLIC_TOOL_ERROR] + return "工具调用失败，请稍后重试" diff --git a/backend/services/brains/workflow_brain.py b/backend/services/brains/workflow_brain.py index 92506e8..8f7631d 100644 --- a/backend/services/brains/workflow_brain.py +++ b/backend/services/brains/workflow_brain.py @@ -11,7 +11,7 @@ from typing import Any from uuid import uuid4 from loguru import logger -from models import AssistantConfig, RuntimeTool +from models import AssistantConfig, RuntimeTool, RuntimeToolBinding from db.session import SessionLocal from pipecat.flows import ( ContextStrategy, @@ -61,7 +61,8 @@ from services.message_stage import ( confirmation_context_message, ) from services.runtime_variables import DynamicVariableError, DynamicVariableStore -from services.system_tools import state_update_properties, system_tool_kind +from services.system_tools import runtime_system_tools, state_update_properties, system_tool_kind +from services.state_expressions import evaluate_update_rules from services.tool_executor import ToolExecutionError, ToolExecutor from services.tool_policy import policy_for_tool from services.workflow.agent import WorkflowAgentStage @@ -654,20 +655,26 @@ class WorkflowBrain(BaseBrain): registered_names.add(function_name) functions.append(function) - for tool_id in stage.tool_ids: - tool = self._tool_by_id.get(str(tool_id)) + for raw_binding in stage.tool_bindings: + binding = RuntimeToolBinding(**raw_binding) + tool = self._tool_by_id.get(binding.tool_id) if not tool: continue - if tool.type == "system": - append_function( - self._workflow_system_tool( - tool, - node_id=node_id, - state_variable_names=stage.state_variable_names, - ) - ) - elif tool.type in {"http", "mcp", "client"}: - append_function(self._flow_tool(tool, node_id)) + if ( + tool.type == "client" + and self._require_runtime().client_tools is not None + and getattr( + self._require_runtime().client_tools, + "supports", + lambda _name: True, + )(tool.function_name) is False + ): + logger.warning(f"跳过客户端未上报的工具: {tool.function_name}") + continue + if tool.type in {"http", "mcp", "client"}: + append_function(self._flow_tool(tool, node_id, binding)) + for tool in runtime_system_tools(stage.built_in_tools): + append_function(self._workflow_system_tool(tool, node_id=node_id)) append_function(self._knowledge_function(node_id)) if stage.vision_enabled and self._require_runtime().vision_function: vision_function = self._require_runtime().vision_function @@ -765,9 +772,13 @@ class WorkflowBrain(BaseBrain): "respond_immediately": False, } - def _flow_tool(self, tool: RuntimeTool, node_id: str) -> FlowsFunctionSchema: - properties, required = self._tools.schema_parts(tool) - self._tools.register_secrets(tool) + def _flow_tool( + self, + tool: RuntimeTool, + node_id: str, + binding: RuntimeToolBinding, + ) -> FlowsFunctionSchema: + properties, required = self._tools.schema_parts(tool, binding) policy = policy_for_tool(tool) async def handler(args, _flow_manager): @@ -784,7 +795,11 @@ class WorkflowBrain(BaseBrain): toolType=tool.type, ) try: - result = await self._tools.execute(tool, dict(args or {})) + result = await self._tools.execute( + tool, + dict(args or {}), + binding=binding, + ) except ToolExecutionError as exc: await self._emit_trace( "tool_failed", @@ -935,7 +950,6 @@ class WorkflowBrain(BaseBrain): tool: RuntimeTool, *, node_id: str, - state_variable_names: tuple[str, ...], ) -> FlowsFunctionSchema: """Build one platform-owned tool scoped to the active Agent node.""" kind = system_tool_kind(tool.definition or {}) @@ -943,7 +957,6 @@ class WorkflowBrain(BaseBrain): schema = self._workflow_update_state_tool( tool, node_id, - state_variable_names=state_variable_names, ) elif kind == "skip_turn": schema = self._workflow_skip_turn_tool(tool) @@ -962,22 +975,19 @@ class WorkflowBrain(BaseBrain): self, tool: RuntimeTool, node_id: str, - *, - state_variable_names: tuple[str, ...], ) -> FlowsFunctionSchema: - allowed = frozenset(state_variable_names) + config = (tool.definition or {}).get("config") or {} + rules = list(config.get("updates") or []) async def handler(args, _flow_manager): - values = dict(args or {}) - unauthorized = sorted(set(values) - allowed) - if unauthorized: - return { - "status": "error", - "message": "状态变量未获当前节点授权: " + ",".join(unauthorized), - } try: + values = evaluate_update_rules( + rules, + dict(self._store.values), + dict(args or {}), + ) changed = self._store.assign_declared_many(values) - except DynamicVariableError as exc: + except (DynamicVariableError, ValueError) as exc: return {"status": "error", "message": f"状态更新失败: {exc}"} result = { "status": "success", @@ -1009,7 +1019,7 @@ class WorkflowBrain(BaseBrain): ), properties=state_update_properties( self._cfg.dynamic_variable_definitions if self._cfg else {}, - allowed_names=state_variable_names, + rules=rules, ), required=[], handler=handler, @@ -1311,11 +1321,9 @@ class WorkflowBrain(BaseBrain): """Apply one deterministic, atomic dynamic-variable update.""" self._state.enter(node_id, WorkflowStatus.RUNNING_ACTION) await self._emit_node_active(node_id) - raw_assignments = self._engine.data(node_id).get("assignments") or {} - rendered = self._store.render_data(raw_assignments) - if not isinstance(rendered, dict): - raise DynamicVariableError("Update State 节点赋值必须是对象") - changed = self._store.assign_declared_many(rendered) + updates = self._engine.data(node_id).get("updates") or [] + values = evaluate_update_rules(updates, dict(self._store.values)) + changed = self._store.assign_declared_many(values) if changed: await self._emit_variables( reason="update_state", @@ -1461,8 +1469,10 @@ class WorkflowBrain(BaseBrain): data = self._engine.data(node_id) runtime = self._require_runtime() invocation_id = self._actions.new_invocation_id() - tool_id = str(data.get("toolId") or "") - tool = self._tool_by_id.get(tool_id) + raw_binding = data.get("toolBinding") or {} + binding = RuntimeToolBinding(**raw_binding) + tool_id = binding.tool_id + tool = self._tool_by_id.get(binding.tool_id) try: stage_result = await self._action_stages.run( ActionStageSpec( @@ -1470,8 +1480,7 @@ class WorkflowBrain(BaseBrain): StageAction( id=node_id, tool=tool, - arguments=data.get("arguments") or {}, - result_assignments=self._action_result_assignments(data), + binding=binding, invocation_id=invocation_id, ), ), @@ -1633,23 +1642,6 @@ class WorkflowBrain(BaseBrain): }[outcome.status] await self._emit_trace(event, nodeId=node_id, outcome=outcome.trace_payload()) - @staticmethod - def _action_result_assignments( - data: dict[str, Any], - ) -> dict[str, str] | None: - """Resolve node mapping semantics for ToolExecutor. - - ``None`` means inherit the reusable tool's mapping, while an empty - dictionary explicitly disables all result assignments. - """ - mode = str(data.get("resultAssignmentMode") or "none") - if mode == "inherit": - return None - if mode == "override": - assignments = data.get("resultAssignments") - return dict(assignments) if isinstance(assignments, dict) else {} - return {} - async def _enter_handoff(self, node_id: str) -> None: self._state.enter(node_id, WorkflowStatus.HANDOFF) await self._emit_node_active(node_id) diff --git a/backend/services/client_tools.py b/backend/services/client_tools.py index d570126..238788f 100644 --- a/backend/services/client_tools.py +++ b/backend/services/client_tools.py @@ -44,6 +44,8 @@ class _DeferredClientToolResult: class ClientToolPort(Protocol): + def supports(self, function_name: str) -> bool: ... + async def call( self, function_name: str, @@ -65,6 +67,13 @@ class ClientToolBroker(FrameProcessor): self._deferred_results: deque[_DeferredClientToolResult] = deque() self._interrupt_handler: Callable[[], Awaitable[None]] | None = None self._closed_message: str | None = None + self._capabilities: frozenset[str] | None = None + + def set_capabilities(self, names: set[str] | None) -> None: + self._capabilities = None if names is None else frozenset(names) + + def supports(self, function_name: str) -> bool: + return self._capabilities is None or function_name in self._capabilities def set_interrupt_handler( self, @@ -85,6 +94,8 @@ class ClientToolBroker(FrameProcessor): ) -> dict[str, Any]: if self._closed_message is not None: raise ClientToolError(self._closed_message) + if not self.supports(function_name): + raise ClientToolError(f"客户端未注册工具: {function_name}") tool_call_id = f"client_{uuid4().hex}" message = { diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 18632bd..a77f0b8 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -19,6 +19,7 @@ from models import ( RuntimeMcpServer, RuntimeModelResource, RuntimeTool, + RuntimeToolBinding, ) from services.node_specs import graph_references, normalize_graph from sqlalchemy import select @@ -90,15 +91,21 @@ def _secret(resource: ModelResource | None, key: str, default: str = "") -> str: return str((resource.secrets or {}).get(key) or default) -async def _bound_tool_ids(session: AsyncSession, assistant_id: str) -> list[str]: +async def _bound_tools( + session: AsyncSession, + assistant_id: str, +) -> list[RuntimeToolBinding]: rows = ( await session.execute( - select(AssistantToolBinding.tool_id) + select(AssistantToolBinding) .where(AssistantToolBinding.assistant_id == assistant_id) .order_by(AssistantToolBinding.created_at, AssistantToolBinding.tool_id) ) ).scalars().all() - return [str(tool_id) for tool_id in rows] + return [ + RuntimeToolBinding(tool_id=row.tool_id, **(row.config or {})) + for row in rows + ] def _startup_tool_ids(startup: dict | None) -> list[str]: @@ -108,16 +115,24 @@ def _startup_tool_ids(startup: dict | None) -> list[str]: for action in (startup or {}).get("actions") or []: if not isinstance(action, dict): continue - tool_id = str(action.get("tool_id") or action.get("toolId") or "").strip() + binding = action.get("tool_binding") or action.get("toolBinding") or {} + tool_id = str(binding.get("tool_id") or binding.get("toolId") or "").strip() if tool_id and tool_id not in result: result.append(tool_id) return result -def _runtime_tool_ids(llm_tool_ids: list[str], startup: dict | None) -> list[str]: +def _runtime_tool_ids( + bindings: list[RuntimeToolBinding], + startup: dict | None, +) -> list[str]: """Merge both tool scopes without changing either scope's meaning.""" - return list(dict.fromkeys([*llm_tool_ids, *_startup_tool_ids(startup)])) + return list( + dict.fromkeys( + [*(binding.tool_id for binding in bindings), *_startup_tool_ids(startup)] + ) + ) async def _tools_for( @@ -251,8 +266,8 @@ async def resolve_runtime_config( if kb.status == "active" } - llm_tool_ids = await _bound_tool_ids(session, assistant.id) - runtime_tool_ids = _runtime_tool_ids(llm_tool_ids, assistant.startup or {}) + prompt_tool_bindings = await _bound_tools(session, assistant.id) + runtime_tool_ids = _runtime_tool_ids(prompt_tool_bindings, assistant.startup or {}) runtime_tools = await _tools_for(session, assistant, runtime_tool_ids) return AssistantConfig( @@ -267,8 +282,9 @@ async def resolve_runtime_config( turnConfig=assistant.turn_config or {}, startup=assistant.startup or {}, analysis_config=assistant.analysis_config or {}, + built_in_tools=assistant.built_in_tools or {}, tools=runtime_tools, - llm_tool_ids=llm_tool_ids, + tool_bindings=prompt_tool_bindings, knowledge_base_id=assistant.knowledge_base_id, knowledge_base_name=knowledge_base.name if knowledge_base else "", knowledge_base_description=knowledge_base.description if knowledge_base else "", diff --git a/backend/services/interface_catalog.py b/backend/services/interface_catalog.py index 55d28c9..a83305c 100644 --- a/backend/services/interface_catalog.py +++ b/backend/services/interface_catalog.py @@ -2,6 +2,7 @@ from __future__ import annotations +from services.realtime_tool_capabilities import QWEN_PROMPT_TOOL_FEATURES from typing import Any @@ -123,6 +124,7 @@ INTERFACE_DEFINITIONS: list[dict] = [ "interface_type": "qwen-audio-realtime", "name": "Alibaba Cloud Qwen-Audio Realtime", "capability": "Realtime", + "features": {"realtimePromptTools": QWEN_PROMPT_TOOL_FEATURES}, "fields": [ field( "modelId", diff --git a/backend/services/node_specs.py b/backend/services/node_specs.py index edf1706..87df107 100644 --- a/backend/services/node_specs.py +++ b/backend/services/node_specs.py @@ -1,4 +1,4 @@ -"""Workflow v3 node catalog, v2 compatibility normalization, and validation.""" +"""Workflow v4 node catalog, normalization, and validation.""" from __future__ import annotations @@ -14,7 +14,7 @@ from services.message_policy import ( ) -SPEC_VERSION = "3" +SPEC_VERSION = "4" NODE_TYPES = { "start", "agent", @@ -28,7 +28,6 @@ EDGE_MODES = {"llm", "expression", "always"} WORKFLOW_RUNTIME_MODES = {"pipeline", "realtime"} WORKFLOW_LLM_ROUTING_MODES = {"llm_router", "edge_tool"} AGENT_ENTRY_MODES = {"wait_user", "generate"} -ACTION_RESULT_ASSIGNMENT_MODES = {"inherit", "override", "none"} ACTION_USER_INPUT_POLICIES = {"queue", "block"} AUTOMATIC_NODE_TYPES = { "start", @@ -195,11 +194,8 @@ def _normalize_agent_data(data: dict[str, Any]) -> None: if data.get("entryMode") not in AGENT_ENTRY_MODES: data["entryMode"] = "wait_user" data.pop("entrySpeech", None) - data.pop("systemTools", None) - state_names = data.get("stateVariableNames") - data["stateVariableNames"] = list( - dict.fromkeys(str(name) for name in state_names or [] if str(name)) - ) + data.setdefault("toolBindings", []) + data.setdefault("builtInTools", {}) legacy_prompt = str(data.pop("visionFocusPrompt", "") or "").strip() if "visionPrompt" not in data: data["visionPrompt"] = legacy_prompt @@ -215,35 +211,30 @@ def _normalize_agent_data(data: dict[str, Any]) -> None: data.get("visionModelResourceId"), data.get("visionPrompt"), data.get("knowledgeBaseId"), - data.get("toolIds"), + data.get("toolBindings"), ) ) data["inheritGlobalConfig"] = not has_node_overrides def _normalize_action_data(data: dict[str, Any]) -> None: - """Add Action defaults while preserving the behavior of saved graphs. - - Older Action nodes always passed an empty mapping when no node-level - assignments were configured, so they did *not* inherit the reusable - tool's assignments. Infer ``none`` for that shape and ``override`` for - an existing non-empty mapping. Newly created nodes should persist their - intended mode explicitly. - """ - if "resultAssignmentMode" not in data: - assignments = data.get("resultAssignments") - data["resultAssignmentMode"] = ( - "override" if isinstance(assignments, dict) and assignments else "none" - ) - data.setdefault("resultAssignments", {}) + """Add deterministic Action defaults.""" + data.setdefault( + "toolBinding", + { + "toolId": "", + "argumentBindings": {}, + "templateBindings": {}, + "resultAssignments": {}, + }, + ) data.setdefault("userInputPolicy", "queue") data.pop("speech", None) def _normalize_update_state_data(data: dict[str, Any]) -> None: - """Keep deterministic state updates as one explicit assignment object.""" - assignments = data.get("assignments") - data["assignments"] = dict(assignments) if isinstance(assignments, dict) else {} + """Keep deterministic state updates as typed expression rules.""" + data.setdefault("updates", []) def _normalize_message_data(data: dict[str, Any]) -> None: @@ -278,7 +269,8 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") -> settings.setdefault("visionPrompt", legacy_prompt) if "visionPromptMode" not in settings: settings["visionPromptMode"] = "guided" if legacy_prompt else "runtime" - settings.setdefault("toolIds", []) + settings.setdefault("toolBindings", []) + settings.setdefault("builtInTools", {}) settings.setdefault("knowledgeBaseId", "") settings.setdefault("knowledgeMode", "automatic") settings.setdefault("knowledgeTopN", 5) @@ -288,7 +280,7 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") -> def normalize_graph(graph: dict[str, Any] | None) -> dict[str, Any]: - """Return a deep-copied v3 graph; preserve v3 IDs and migrate v2 semantics.""" + """Return a deep-copied v4 graph.""" source = deepcopy(graph or {}) if str(source.get("specVersion") or "") == SPEC_VERSION: settings = source.setdefault("settings", {}) @@ -402,7 +394,7 @@ def normalize_graph(graph: dict[str, Any] | None) -> dict[str, Any]: settings = deepcopy(source.get("settings") or {}) _normalize_settings(settings, global_prompt=global_prompt) return { - "specVersion": 3, + "specVersion": 4, "settings": settings, "nodes": mapped_nodes, "edges": mapped_edges, @@ -475,12 +467,11 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: entry_mode = data.get("entryMode", "wait_user") if entry_mode not in AGENT_ENTRY_MODES: errors.append(f"Agent 节点 {node_id} 的进入模式无效:{entry_mode}") - state_names = data.get("stateVariableNames", []) - if not isinstance(state_names, list) or any( - not isinstance(name, str) for name in state_names - ): - errors.append(f"Agent 节点 {node_id} 的状态变量授权必须是列表") source = settings if data.get("inheritGlobalConfig", True) else data + if not isinstance(source.get("toolBindings"), list): + errors.append(f"Agent 节点 {node_id} 的工具绑定必须是列表") + if not isinstance(source.get("builtInTools"), dict): + errors.append(f"Agent 节点 {node_id} 的内建工具配置必须是对象") vision_prompt_mode = str(source.get("visionPromptMode") or "runtime") if vision_prompt_mode not in VISION_PROMPT_MODES: errors.append( @@ -562,13 +553,9 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: ) elif node_type == "action": data = node.get("data") or {} - assignment_mode = data.get("resultAssignmentMode") - if assignment_mode not in ACTION_RESULT_ASSIGNMENT_MODES: - errors.append( - f"Action 节点 {node_id} 的结果变量模式无效:{assignment_mode}" - ) - if not isinstance(data.get("resultAssignments"), dict): - errors.append(f"Action 节点 {node_id} 的结果变量映射必须是对象") + binding = data.get("toolBinding") + if not isinstance(binding, dict) or not binding.get("toolId"): + errors.append(f"Action 节点 {node_id} 必须配置工具绑定") input_policy = data.get("userInputPolicy") if input_policy not in ACTION_USER_INPUT_POLICIES: errors.append( @@ -580,10 +567,10 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: ) elif node_type == "update_state": data = node.get("data") or {} - assignments = data.get("assignments") - if not isinstance(assignments, dict) or not assignments: + updates = data.get("updates") + if not isinstance(updates, list) or not updates: errors.append( - f"Update State 节点 {node_id} 必须配置至少一个变量赋值" + f"Update State 节点 {node_id} 必须配置至少一条更新规则" ) if counts["start"] != 1: @@ -729,7 +716,11 @@ def graph_references(graph: dict[str, Any]) -> dict[str, set[str]]: ) if value } - tools: set[str] = {str(tool_id) for tool_id in settings.get("toolIds") or []} + tools: set[str] = { + str(binding.get("toolId") or "") + for binding in settings.get("toolBindings") or [] + if isinstance(binding, dict) and binding.get("toolId") + } knowledge: set[str] = ( {str(settings["knowledgeBaseId"])} if settings.get("knowledgeBaseId") @@ -753,10 +744,12 @@ def graph_references(graph: dict[str, Any]) -> dict[str, set[str]]: ): if resource_id: resources.add(str(resource_id)) - for tool_id in data.get("toolIds") or []: - tools.add(str(tool_id)) + for binding in data.get("toolBindings") or []: + if isinstance(binding, dict) and binding.get("toolId"): + tools.add(str(binding["toolId"])) if data.get("knowledgeBaseId"): knowledge.add(str(data["knowledgeBaseId"])) - if data.get("toolId"): - tools.add(str(data["toolId"])) + binding = data.get("toolBinding") + if isinstance(binding, dict) and binding.get("toolId"): + tools.add(str(binding["toolId"])) return {"model_resources": resources, "tools": tools, "knowledge_bases": knowledge} diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index ad1da7a..2227bb4 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -250,6 +250,7 @@ async def run_pipeline( assistant_id: str | None = None, channel: str = "webrtc", protocol_adapter: PipelineProtocolAdapter | None = None, + client_tool_names: set[str] | None = None, ) -> None: """在给定 transport 上构建并运行管线,直到连接结束。 @@ -469,6 +470,7 @@ async def run_pipeline( should_ignore_update=lambda: call_end.ending, ) client_tools = ClientToolBroker() + client_tools.set_capabilities(client_tool_names) # Confirmation nodes mute the user aggregator, which intentionally drops # InterruptionFrame. Relay confirmation interruptions from the next # processor instead so LLM, assistant aggregation, TTS, and output all see @@ -1119,6 +1121,7 @@ async def run_realtime_pipeline( on_state_change=queue_call_end_state, ) client_tools = ClientToolBroker() + client_tools.set_capabilities(client_tool_names) async def interrupt_for_client_tool_result() -> None: """Release confirmations after the realtime response is interrupted. diff --git a/backend/services/pipecat/processors.py b/backend/services/pipecat/processors.py index 42233bd..c5d4f99 100644 --- a/backend/services/pipecat/processors.py +++ b/backend/services/pipecat/processors.py @@ -561,6 +561,10 @@ class RealtimeDynamicVariableProcessor(FrameProcessor): if content: self._brain.record_user_message(content) await self._refresh_instructions() + elif event_type == "assistant-text-start": + await self._brain.on_assistant_text_start( + str(message.get("turn_id") or "") + ) elif event_type == "assistant-text-end": await self._brain.on_assistant_text_end( str(message.get("turn_id") or ""), diff --git a/backend/services/pipecat/qwen_audio_realtime.py b/backend/services/pipecat/qwen_audio_realtime.py index a082550..ecdc7bc 100644 --- a/backend/services/pipecat/qwen_audio_realtime.py +++ b/backend/services/pipecat/qwen_audio_realtime.py @@ -52,6 +52,7 @@ SUPPORTED_TURN_DETECTION_MODES = frozenset({"none", "server_vad", "smart_turn"}) ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None] SpeechStartedHandler = Callable[[], Awaitable[None]] +ResponseDoneHandler = Callable[[str, bool], Awaitable[None]] class QwenAudioRealtimeService(AIService): @@ -117,6 +118,7 @@ class QwenAudioRealtimeService(AIService): self._warned_input_sample_rate = False self._response_active = False + self._active_response_id = "" self._audio_suppressed = False self._assistant_turn_id: str | None = None self._assistant_text = "" @@ -130,12 +132,14 @@ class QwenAudioRealtimeService(AIService): self._tools: list[RealtimeTool] = [] self._tool_session = RealtimeToolSession( self._send_tool_event, - wait_for_response_boundary=self._response_done.wait, + suppress_response=self._cancel_active_response, ) self._fixed_speech_completion: asyncio.Future[None] | None = None self._suppress_response_transcript = False self._speech_started_handler: SpeechStartedHandler | None = None + self._response_done_handler: ResponseDoneHandler | None = None self._function_names: dict[str, str] = {} + self._responses_with_tool_calls: set[str] = set() async def start(self, frame: StartFrame) -> None: await super().start(frame) @@ -201,6 +205,7 @@ class QwenAudioRealtimeService(AIService): return if isinstance(frame, InterruptionFrame): + self._tool_session.invalidate_pending_follow_ups() await self._cancel_active_response() await self.push_frame(frame, direction) @@ -209,6 +214,7 @@ class QwenAudioRealtimeService(AIService): """Append text to Qwen's conversation and optionally start a response.""" if not text: return + self._tool_session.invalidate_pending_follow_ups() await self._send_event( { "type": "conversation.item.create", @@ -224,6 +230,7 @@ class QwenAudioRealtimeService(AIService): async def interrupt(self) -> None: """Cancel current model output and notify the rest of the pipeline.""" + self._tool_session.invalidate_pending_follow_ups() await self._cancel_active_response() await self.broadcast_interruption() @@ -268,6 +275,12 @@ class QwenAudioRealtimeService(AIService): ) -> None: self._speech_started_handler = handler + def set_response_done_handler( + self, + handler: ResponseDoneHandler | None, + ) -> None: + self._response_done_handler = handler + async def speak(self, text: str) -> None: """Ask Qwen to speak a fixed greeting, then remove the hidden request. @@ -345,7 +358,7 @@ class QwenAudioRealtimeService(AIService): "type": "session.update", "session": { "instructions": instructions, - "tools": [tool.provider_schema() for tool in tools], + "tools": [tool.qwen_schema() for tool in tools], "tool_choice": "auto", }, }, @@ -375,7 +388,7 @@ class QwenAudioRealtimeService(AIService): "output_audio_format": "pcm", "turn_detection": self._turn_detection_config(), "max_history_turns": self._max_history_turns, - "tools": [tool.provider_schema() for tool in self._tools], + "tools": [tool.qwen_schema() for tool in self._tools], "tool_choice": "auto", } @@ -427,12 +440,14 @@ class QwenAudioRealtimeService(AIService): self._response_done.set() self._pending_events.clear() self._response_active = False + self._active_response_id = "" self._user_transcript_pending = False self._user_transcript_item_id = "" self._user_transcript_timestamp = "" self._deferred_assistant_messages.clear() - self._tool_session.clear() + await self._tool_session.shutdown() self._function_names.clear() + self._responses_with_tool_calls.clear() self._entry_trigger_item_id = None self._resolve_fixed_speech() if websocket and websocket.state is State.OPEN: @@ -458,6 +473,8 @@ class QwenAudioRealtimeService(AIService): self._websocket = None self._session_ready.clear() self._response_done.set() + await self._tool_session.shutdown() + self._responses_with_tool_calls.clear() self._greeting_request_item_id = None self._entry_trigger_item_id = None self._resolve_fixed_speech() @@ -481,6 +498,10 @@ class QwenAudioRealtimeService(AIService): await self._send_event(payload, wait_until_ready=False) elif event_type == "response.created": self._response_active = True + response = event.get("response") + self._active_response_id = str( + response.get("id") if isinstance(response, dict) else "" + ) self._response_done.clear() self._audio_suppressed = False elif event_type == "response.audio.delta": @@ -513,6 +534,7 @@ class QwenAudioRealtimeService(AIService): await self._release_deferred_assistant_messages() elif event_type == "input_audio_buffer.speech_started": user_turn_timestamp = time_now_iso8601() + self._tool_session.invalidate_pending_follow_ups() await self._cancel_active_response() await self.broadcast_interruption() if self._speech_started_handler is not None: @@ -526,7 +548,13 @@ class QwenAudioRealtimeService(AIService): elif event_type == "response.done": response = event.get("response") status = response.get("status") if isinstance(response, dict) else None + response_id = str( + (response.get("id") if isinstance(response, dict) else "") + or event.get("response_id") + or self._active_response_id + ) interrupted = status in {"cancelled", "incomplete", "interrupted", "failed"} + response_content = self._assistant_text self._response_active = False await self.push_frame(TTSStoppedFrame()) await self._finish_assistant_text(interrupted=interrupted) @@ -534,16 +562,22 @@ class QwenAudioRealtimeService(AIService): await self._delete_entry_trigger() self._resolve_fixed_speech() self._response_done.set() + self._tool_session.finish_response( + response_id, + cancelled=interrupted, + ) + had_tool_calls = response_id in self._responses_with_tool_calls + self._responses_with_tool_calls.discard(response_id) + if self._response_done_handler is not None and not had_tool_calls: + await self._response_done_handler(response_content, interrupted) + self._active_response_id = "" elif event_type == "response.output_item.added": self._remember_function_call(event) elif event_type in { "response.function_call_arguments.done", "response.output_item.done", }: - self.create_task( - self._handle_function_call_event(dict(event)), - name="qwen_audio_realtime_tool_call", - ) + await self._handle_function_call_event(dict(event)) elif event_type == "error": error = event.get("error") message = error.get("message") if isinstance(error, dict) else str(error) @@ -591,18 +625,17 @@ class QwenAudioRealtimeService(AIService): ) if not name: return - try: - await self._tool_session.handle_call( - name=name, - call_id=call_id, - arguments=source.get("arguments", event.get("arguments")), - ) - except asyncio.CancelledError: - raise - except Exception as exc: # noqa: BLE001 - background provider task - logger.exception(f"Qwen-Audio Realtime 工具调用处理失败:{exc}") - finally: - self._function_names.pop(call_id, None) + response_id = str(event.get("response_id") or self._active_response_id) + if response_id: + self._responses_with_tool_calls.add(response_id) + self._tool_session.submit_call( + response_id=response_id, + name=name, + call_id=call_id, + arguments=source.get("arguments", event.get("arguments")), + output_index=int(event.get("output_index") or 0), + ) + self._function_names.pop(call_id, None) def _remember_function_call(self, event: dict[str, Any]) -> None: item = event.get("item") diff --git a/backend/services/pipecat/realtime_tools.py b/backend/services/pipecat/realtime_tools.py index dcbee19..5c1272d 100644 --- a/backend/services/pipecat/realtime_tools.py +++ b/backend/services/pipecat/realtime_tools.py @@ -4,23 +4,29 @@ from __future__ import annotations import asyncio import json +from collections import deque from collections.abc import Awaitable, Callable from dataclasses import dataclass, field -from typing import Any +from typing import Any, Literal from loguru import logger +RealtimeToolFollowUp = Literal["continue", "none", "suppress"] + + @dataclass(frozen=True) class RealtimeTool: - """Small JSON-schema tool definition understood by both providers.""" + """Provider-neutral JSON-schema function definition.""" name: str description: str properties: dict[str, Any] = field(default_factory=dict) required: tuple[str, ...] = () - def provider_schema(self) -> dict[str, Any]: + def openai_schema(self) -> dict[str, Any]: + """Serialize the flat schema used by OpenAI-compatible providers.""" + return { "type": "function", "name": self.name, @@ -32,13 +38,26 @@ class RealtimeTool: }, } + def qwen_schema(self) -> dict[str, Any]: + """Serialize Qwen-Audio's documented nested function schema.""" + + flat = self.openai_schema() + return { + "type": "function", + "function": { + "name": flat["name"], + "description": flat["description"], + "parameters": flat["parameters"], + }, + } + @dataclass(frozen=True) class RealtimeToolResult: - """Tool output plus whether the model should continue immediately.""" + """Tool output and the action taken after every call in one response.""" output: dict[str, Any] - continue_response: bool = True + follow_up: RealtimeToolFollowUp = "continue" after_output: Callable[[], Awaitable[None]] | None = None @@ -46,96 +65,271 @@ RealtimeToolDispatcher = Callable[ [str, dict[str, Any], str], Awaitable[RealtimeToolResult] ] SendProviderEvent = Callable[[dict[str, Any]], Awaitable[None]] -WaitForResponseBoundary = Callable[[], Awaitable[Any]] +SuppressProviderResponse = Callable[[], Awaitable[None]] + + +@dataclass(frozen=True) +class _PendingCall: + name: str + call_id: str + arguments: dict[str, Any] + output_index: int + argument_error: str = "" + + +@dataclass +class _ResponseCalls: + calls: dict[str, _PendingCall] = field(default_factory=dict) + cancelled: bool = False + processing: bool = False class RealtimeToolSession: - """Serialize provider calls and answer every call id at most once.""" + """Coordinate all function calls emitted by one provider response. + + Qwen may emit more than one function call in a response. Calls are collected + until ``response.done``, executed in output order, and followed by at most + one new model response. Duplicate argument/output-item events are ignored by + call id. + """ def __init__( self, send_event: SendProviderEvent, *, - wait_for_response_boundary: WaitForResponseBoundary | None = None, + suppress_response: SuppressProviderResponse | None = None, ) -> None: self._send_event = send_event - self._wait_for_response_boundary = wait_for_response_boundary + self._suppress_response = suppress_response self._dispatcher: RealtimeToolDispatcher | None = None self._handled_call_ids: set[str] = set() - self._lock = asyncio.Lock() + self._handled_call_order: deque[str] = deque() + self._finished_response_ids: set[str] = set() + self._finished_response_order: deque[str] = deque() + self._responses: dict[str, _ResponseCalls] = {} + self._tasks: set[asyncio.Task[None]] = set() def set_dispatcher(self, dispatcher: RealtimeToolDispatcher | None) -> None: self._dispatcher = dispatcher - async def handle_call( + def submit_call( self, *, + response_id: str, name: str, call_id: str, arguments: str | dict[str, Any] | None, + output_index: int = 0, ) -> None: + """Register one complete call without yielding to the receive loop.""" + if not call_id: logger.warning("Realtime function call 缺少 call_id，已忽略") return - async with self._lock: - if call_id in self._handled_call_ids: - return - self._handled_call_ids.add(call_id) - parsed = self._parse_arguments(arguments) - try: - if self._dispatcher is None: - result = RealtimeToolResult( - {"status": "error", "message": "当前会话未注册工具处理器"} - ) - else: - result = await self._dispatcher(name, parsed, call_id) - except Exception as exc: # noqa: BLE001 - return tool errors to provider - logger.exception(f"Realtime 工具 {name} 执行失败:{exc}") - result = RealtimeToolResult( - { - "status": "error", - "message": f"工具执行失败:{type(exc).__name__}", - } - ) + if call_id in self._handled_call_ids: + return + self._handled_call_ids.add(call_id) + self._handled_call_order.append(call_id) + if len(self._handled_call_order) > 4096: + expired_call_id = self._handled_call_order.popleft() + self._handled_call_ids.discard(expired_call_id) + group_id = response_id or f"response_{call_id}" + if group_id in self._finished_response_ids: + return + group = self._responses.setdefault(group_id, _ResponseCalls()) + if group.cancelled or group.processing: + return + parsed_arguments, argument_error = self._parse_arguments(arguments) + group.calls[call_id] = _PendingCall( + name=name, + call_id=call_id, + arguments=parsed_arguments, + output_index=output_index, + argument_error=argument_error, + ) - await self._send_event( + def finish_response(self, response_id: str, *, cancelled: bool = False) -> None: + """Seal one response and process its calls in a background task.""" + + if not response_id: + if len(self._responses) != 1: + return + response_id = next(iter(self._responses)) + self._remember_finished_response(response_id) + group = self._responses.get(response_id) + if group is None: + return + group.cancelled = group.cancelled or cancelled + if group.processing: + return + group.processing = True + task = asyncio.create_task( + self._process_response(response_id, group), + name=f"realtime_tools_{response_id}", + ) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + + async def wait_pending(self) -> None: + """Wait for current batches; primarily useful for orderly shutdown/tests.""" + + tasks = tuple(self._tasks) + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + def cancel_response(self, response_id: str) -> None: + """Prevent unstarted calls and any follow-up for a stale response.""" + + group = self._responses.get(response_id) + if group is not None: + group.cancelled = True + + def invalidate_pending_follow_ups(self) -> None: + """Make in-flight tool groups stale without cancelling active I/O.""" + + for group in self._responses.values(): + group.cancelled = True + + def cancel_all(self) -> None: + """Cancel pending work when the provider connection closes.""" + + for group in self._responses.values(): + group.cancelled = True + for task in tuple(self._tasks): + task.cancel() + self._responses.clear() + + async def _process_response( + self, + response_id: str, + group: _ResponseCalls, + ) -> None: + try: + results: list[RealtimeToolResult] = [] + calls = sorted( + group.calls.values(), + key=lambda item: (item.output_index, item.call_id), + ) + for call in calls: + if group.cancelled: + await self._send_output( + call.call_id, + {"status": "cancelled", "message": "调用已被新的用户输入取代"}, + ) + continue + result = await self._dispatch(call) + results.append(result) + await self._send_output(call.call_id, result.output) + + if group.cancelled: + return + for result in results: + if result.after_output is not None: + try: + await result.after_output() + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - isolate callbacks + logger.exception(f"Realtime 工具结果回调失败:{exc}") + follow_up = self._combined_follow_up(results) + if follow_up == "suppress" and self._suppress_response is not None: + await self._suppress_response() + elif follow_up == "continue": + await self._send_event({"type": "response.create"}) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - background provider task + logger.exception(f"Realtime response 工具处理失败:{exc}") + finally: + self._responses.pop(response_id, None) + + async def _send_output(self, call_id: str, output: dict[str, Any]) -> None: + await self._send_event( + { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": call_id, + "output": json.dumps( + output, + ensure_ascii=False, + separators=(",", ":"), + ), + }, + } + ) + + async def _dispatch(self, call: _PendingCall) -> RealtimeToolResult: + if call.argument_error: + return RealtimeToolResult( + {"status": "error", "message": call.argument_error} + ) + if self._dispatcher is None: + return RealtimeToolResult( + {"status": "error", "message": "当前会话未注册工具处理器"} + ) + try: + return await self._dispatcher( + call.name, + call.arguments, + call.call_id, + ) + except Exception as exc: # noqa: BLE001 - return stable errors to provider + logger.exception(f"Realtime 工具 {call.name} 执行失败:{exc}") + return RealtimeToolResult( { - "type": "conversation.item.create", - "item": { - "type": "function_call_output", - "call_id": call_id, - "output": json.dumps( - result.output, - ensure_ascii=False, - separators=(",", ":"), - ), - }, + "status": "error", + "message": f"工具执行失败:{type(exc).__name__}", } ) - # Function-call argument events arrive before the provider's - # response.done boundary. Starting another response sooner is - # rejected by both supported realtime providers. The provider - # receive loop stays free while this background task waits. - if self._wait_for_response_boundary is not None: - await self._wait_for_response_boundary() - if result.after_output is not None: - await result.after_output() - if result.continue_response: - await self._send_event({"type": "response.create"}) + + @staticmethod + def _combined_follow_up( + results: list[RealtimeToolResult], + ) -> RealtimeToolFollowUp: + actions = {result.follow_up for result in results} + if "suppress" in actions: + return "suppress" + if "continue" in actions: + return "continue" + return "none" def clear(self) -> None: + self.cancel_all() self._handled_call_ids.clear() + self._handled_call_order.clear() + self._finished_response_ids.clear() + self._finished_response_order.clear() + + async def shutdown(self) -> None: + """Cancel and join background calls before releasing a connection.""" + + tasks = tuple(self._tasks) + self.clear() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + def _remember_finished_response(self, response_id: str) -> None: + if response_id in self._finished_response_ids: + return + self._finished_response_ids.add(response_id) + self._finished_response_order.append(response_id) + if len(self._finished_response_order) > 512: + expired = self._finished_response_order.popleft() + self._finished_response_ids.discard(expired) @staticmethod def _parse_arguments( arguments: str | dict[str, Any] | None, - ) -> dict[str, Any]: + ) -> tuple[dict[str, Any], str]: if isinstance(arguments, dict): - return dict(arguments) - if not arguments: - return {} + return dict(arguments), "" + if arguments is None or arguments == "": + return {}, "" try: parsed = json.loads(arguments) except (TypeError, json.JSONDecodeError): - return {} - return dict(parsed) if isinstance(parsed, dict) else {} + return {}, "工具参数不是有效的 JSON 对象" + if not isinstance(parsed, dict): + return {}, "工具参数必须是 JSON 对象" + return dict(parsed), "" diff --git a/backend/services/pipecat/stepfun_realtime.py b/backend/services/pipecat/stepfun_realtime.py index 83431f7..fd09320 100644 --- a/backend/services/pipecat/stepfun_realtime.py +++ b/backend/services/pipecat/stepfun_realtime.py @@ -79,12 +79,12 @@ class StepFunRealtimeService(AIService): self._response_done.set() self._pending_events: list[dict[str, Any]] = [] self._assistant_turn_id: str | None = None + self._active_response_id = "" self._assistant_text = "" self._assistant_timestamp = "" self._tools: list[RealtimeTool] = [] self._tool_session = RealtimeToolSession( self._send_tool_event, - wait_for_response_boundary=self._response_done.wait, ) self._fixed_speech_completion: asyncio.Future[None] | None = None self._fixed_speech_request_item_id: str | None = None @@ -151,6 +151,7 @@ class StepFunRealtimeService(AIService): await self.send_text(text, run_immediately=frame.run_llm is not False) return if isinstance(frame, InterruptionFrame): + self._tool_session.invalidate_pending_follow_ups() await self._send_event({"type": "response.cancel"}, wait_until_ready=False) await self._finish_assistant_text(interrupted=True) self._resolve_fixed_speech() @@ -158,6 +159,7 @@ class StepFunRealtimeService(AIService): await self.push_frame(frame, direction) async def send_text(self, text: str, *, run_immediately: bool = True) -> None: + self._tool_session.invalidate_pending_follow_ups() await self._send_event( { "type": "conversation.item.create", @@ -172,6 +174,7 @@ class StepFunRealtimeService(AIService): await self._send_event({"type": "response.create"}) async def interrupt(self) -> None: + self._tool_session.invalidate_pending_follow_ups() await self._send_event({"type": "response.cancel"}, wait_until_ready=False) await self._finish_assistant_text(interrupted=True) self._resolve_fixed_speech() @@ -291,7 +294,7 @@ class StepFunRealtimeService(AIService): self._websocket = None self._session_ready.clear() self._response_done.set() - self._tool_session.clear() + await self._tool_session.shutdown() self._function_names.clear() self._fixed_speech_request_item_id = None self._entry_trigger_item_id = None @@ -328,6 +331,7 @@ class StepFunRealtimeService(AIService): self._websocket = None self._session_ready.clear() self._response_done.set() + await self._tool_session.shutdown() self._fixed_speech_request_item_id = None self._entry_trigger_item_id = None self._resolve_fixed_speech() @@ -344,6 +348,10 @@ class StepFunRealtimeService(AIService): for payload in pending: await self._send_event(payload, wait_until_ready=False) elif event_type == "response.created": + response = event.get("response") + self._active_response_id = str( + response.get("id") if isinstance(response, dict) else "" + ) self._response_done.clear() elif event_type == "response.audio.delta": audio = event.get("delta") @@ -369,6 +377,7 @@ class StepFunRealtimeService(AIService): elif event_type == "conversation.item.input_audio_transcription.completed": await self._send_transcript("user", str(event.get("transcript") or "")) elif event_type == "input_audio_buffer.speech_started": + self._tool_session.invalidate_pending_follow_ups() await self._send_event({"type": "response.cancel"}, wait_until_ready=False) await self.broadcast_interruption() self._resolve_fixed_speech() @@ -387,16 +396,23 @@ class StepFunRealtimeService(AIService): await self._delete_entry_trigger() self._resolve_fixed_speech() self._response_done.set() + response_id = str( + (response.get("id") if isinstance(response, dict) else "") + or event.get("response_id") + or self._active_response_id + ) + self._tool_session.finish_response( + response_id, + cancelled=interrupted, + ) + self._active_response_id = "" elif event_type == "response.output_item.added": self._remember_function_call(event) elif event_type in { "response.function_call_arguments.done", "response.output_item.done", }: - self.create_task( - self._handle_function_call_event(dict(event)), - name="stepfun_realtime_tool_call", - ) + await self._handle_function_call_event(dict(event)) elif event_type == "error": error = event.get("error") message = error.get("message") if isinstance(error, dict) else str(error) @@ -415,7 +431,7 @@ class StepFunRealtimeService(AIService): "input_audio_format": "pcm16", "output_audio_format": "pcm16", "turn_detection": self._turn_detection_config(), - "tools": [tool.provider_schema() for tool in self._tools], + "tools": [tool.openai_schema() for tool in self._tools], "tool_choice": "auto", }, }, @@ -458,7 +474,7 @@ class StepFunRealtimeService(AIService): "type": "session.update", "session": { "instructions": instructions, - "tools": [tool.provider_schema() for tool in tools], + "tools": [tool.openai_schema() for tool in tools], "tool_choice": "auto", }, }, @@ -493,18 +509,14 @@ class StepFunRealtimeService(AIService): ) if not name: return - try: - await self._tool_session.handle_call( - name=name, - call_id=call_id, - arguments=source.get("arguments", event.get("arguments")), - ) - except asyncio.CancelledError: - raise - except Exception as exc: # noqa: BLE001 - background provider task - logger.exception(f"StepFun Realtime 工具调用处理失败:{exc}") - finally: - self._function_names.pop(call_id, None) + self._tool_session.submit_call( + response_id=str(event.get("response_id") or self._active_response_id), + name=name, + call_id=call_id, + arguments=source.get("arguments", event.get("arguments")), + output_index=int(event.get("output_index") or 0), + ) + self._function_names.pop(call_id, None) def _remember_function_call(self, event: dict[str, Any]) -> None: item = event.get("item") diff --git a/backend/services/realtime/launcher.py b/backend/services/realtime/launcher.py index 04f93f4..ef67184 100644 --- a/backend/services/realtime/launcher.py +++ b/backend/services/realtime/launcher.py @@ -10,6 +10,7 @@ from services.config_resolver import resolve_runtime_config from services.node_specs import graph_references from services.runtime_variables import prepare_dynamic_config from services.workflow_engine import WorkflowEngine +from services.realtime_tool_capabilities import validate_runtime_prompt_tools async def resolve_assistant_config( @@ -137,6 +138,33 @@ def validate_runtime_requirements(config: AssistantConfig) -> None: ("Realtime apiKey", config.realtime_api_key), ], ) + if config.type == "prompt": + tool_by_id = {tool.id: tool for tool in config.tools} + missing_tool_ids = sorted( + { + binding.tool_id + for binding in config.tool_bindings + if binding.tool_id not in tool_by_id + } + ) + if missing_tool_ids: + raise ValueError( + "Prompt Realtime 引用了缺失工具:" + ",".join(missing_tool_ids) + ) + validate_runtime_prompt_tools( + interface_type=config.realtime_interface_type, + external_types={ + tool_by_id[binding.tool_id].type + for binding in config.tool_bindings + if binding.tool_id in tool_by_id + }, + external_function_names={ + tool_by_id[binding.tool_id].function_name + for binding in config.tool_bindings + if binding.tool_id in tool_by_id + }, + built_in_tools=config.built_in_tools, + ) return _validate_voice_resource( diff --git a/backend/services/realtime_tool_capabilities.py b/backend/services/realtime_tool_capabilities.py new file mode 100644 index 0000000..f633e89 --- /dev/null +++ b/backend/services/realtime_tool_capabilities.py @@ -0,0 +1,65 @@ +"""Capability gates for Prompt tools in speech-to-speech runtimes.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from services.system_tools import RESERVED_TOOL_NAMES + + +QWEN_PROMPT_TOOL_FEATURES = { + "externalTypes": ["http", "mcp", "client"], + "builtIns": [ + "endConversation", + "updateState", + "requestHumanHandoff", + ], +} + + +def realtime_prompt_tool_features(interface_type: str) -> dict[str, list[str]]: + """Return the server-trusted runtime capabilities for one provider.""" + + if interface_type == "qwen-audio-realtime": + return { + key: list(values) for key, values in QWEN_PROMPT_TOOL_FEATURES.items() + } + return {"externalTypes": [], "builtIns": []} + + +def validate_runtime_prompt_tools( + *, + interface_type: str, + external_types: set[str], + built_in_tools: Mapping[str, object] | None, + external_function_names: set[str] | None = None, +) -> None: + """Reject stale or inline runtime configurations before opening a socket.""" + + features = realtime_prompt_tool_features(interface_type) + unsupported_external = external_types - set(features["externalTypes"]) + reserved_external = set(external_function_names or set()) & RESERVED_TOOL_NAMES + aliases = { + "end_conversation": "endConversation", + "update_state": "updateState", + "skip_turn": "skipTurn", + "request_human_handoff": "requestHumanHandoff", + } + requested_built_ins = { + aliases.get(str(name), str(name)) + for name, config in (built_in_tools or {}).items() + if config is not None + } + unsupported_built_ins = requested_built_ins - set(features["builtIns"]) + if unsupported_external or unsupported_built_ins or reserved_external: + details = [] + if unsupported_external: + details.append("外部工具:" + ",".join(sorted(unsupported_external))) + if unsupported_built_ins: + details.append("内建工具:" + ",".join(sorted(unsupported_built_ins))) + if reserved_external: + details.append("外部工具使用平台保留名:" + ",".join(sorted(reserved_external))) + raise ValueError( + f"Realtime 接口 {interface_type or 'unknown'} 不支持 " + + "；".join(details) + ) diff --git a/backend/services/runtime_variables.py b/backend/services/runtime_variables.py index 324519b..0382939 100644 --- a/backend/services/runtime_variables.py +++ b/backend/services/runtime_variables.py @@ -181,6 +181,13 @@ class DynamicVariableStore: } return deepcopy(value) + def value(self, name: str) -> Primitive: + """Read one session value for an explicit ToolBinding source.""" + + if name.startswith("secret__"): + raise DynamicVariableError("ToolBinding 不能读取密钥变量") + return self._resolve(name, allow_secrets=False) + def record(self, role: str, content: str, *, completed_agent_turn: bool = False) -> None: if content: self.history.append({"role": role, "message": content}) diff --git a/backend/services/state_expressions.py b/backend/services/state_expressions.py new file mode 100644 index 0000000..ff650eb --- /dev/null +++ b/backend/services/state_expressions.py @@ -0,0 +1,329 @@ +"""Typed, side-effect-free expressions used by built-in state updates. + +Expressions always read one immutable snapshot. Callers validate first, then +evaluate every rule, and only commit after all values succeed. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Mapping + + +MAX_EXPRESSION_DEPTH = 8 +MAX_UPDATE_RULES = 10 +MAX_LLM_PARAMETERS = 10 + + +class StateExpressionError(ValueError): + pass + + +def _field(value: Mapping[str, Any], snake: str, camel: str | None = None) -> Any: + if snake in value: + return value[snake] + return value.get(camel or snake) + + +def _value_type(value: Any) -> str: + if isinstance(value, bool): + return "boolean" + if isinstance(value, (int, float)): + return "number" + if isinstance(value, str): + return "string" + raise StateExpressionError("状态表达式只支持字符串、数字或布尔值") + + +def _definition_types( + definitions: Mapping[str, Mapping[str, Any]] | None, +) -> dict[str, str]: + result: dict[str, str] = {} + for name, definition in (definitions or {}).items(): + value_type = ( + (definition or {}).get("type") + if isinstance(definition, Mapping) + else getattr(definition, "type", None) + ) + result[str(name)] = str(value_type or "string") + return result + + +@dataclass(frozen=True) +class LlmParameter: + name: str + value_type: str + description: str + + +def _inspect_expression( + expression: Mapping[str, Any], + variable_types: Mapping[str, str], + *, + allow_llm: bool, + depth: int, + parameters: dict[str, LlmParameter], +) -> str: + if depth > MAX_EXPRESSION_DEPTH: + raise StateExpressionError( + f"状态表达式深度不能超过 {MAX_EXPRESSION_DEPTH}" + ) + kind = str(expression.get("type") or "") + if kind == "constant": + return _value_type(expression.get("value")) + if kind == "variable": + name = str(_field(expression, "variable_name", "variableName") or "") + if name not in variable_types: + raise StateExpressionError(f"状态表达式引用了未声明变量: {name}") + return variable_types[name] + if kind == "llm": + if not allow_llm: + raise StateExpressionError("确定性 Update State 节点不能使用 LLM 提取值") + name = str(_field(expression, "parameter_name", "parameterName") or "") + value_type = str(_field(expression, "value_type", "valueType") or "string") + description = str(expression.get("description") or "").strip() + if not name or value_type not in {"string", "number", "boolean"}: + raise StateExpressionError("LLM 提取参数名称或类型无效") + if not description: + raise StateExpressionError(f"LLM 提取参数 {name} 必须填写说明") + current = LlmParameter(name, value_type, description) + previous = parameters.get(name) + if previous is not None and previous != current: + raise StateExpressionError(f"LLM 参数 {name} 存在冲突定义") + parameters[name] = current + if len(parameters) > MAX_LLM_PARAMETERS: + raise StateExpressionError( + f"LLM 提取参数不能超过 {MAX_LLM_PARAMETERS} 个" + ) + return value_type + if kind == "if": + condition = expression.get("condition") + then = expression.get("then") + otherwise = _field(expression, "otherwise") + if not all(isinstance(item, Mapping) for item in (condition, then, otherwise)): + raise StateExpressionError("if 表达式缺少 condition/then/otherwise") + condition_type = _inspect_expression( + condition, + variable_types, + allow_llm=allow_llm, + depth=depth + 1, + parameters=parameters, + ) + if condition_type != "boolean": + raise StateExpressionError("if 条件必须是布尔值") + then_type = _inspect_expression( + then, + variable_types, + allow_llm=allow_llm, + depth=depth + 1, + parameters=parameters, + ) + otherwise_type = _inspect_expression( + otherwise, + variable_types, + allow_llm=allow_llm, + depth=depth + 1, + parameters=parameters, + ) + if then_type != otherwise_type: + raise StateExpressionError("if 的两个结果分支类型必须一致") + return then_type + + operands = expression.get("operands") + if kind not in { + "and", "or", "eq", "neq", "gt", "gte", "lt", "lte", + "add", "subtract", "multiply", "divide", + } or not isinstance(operands, list) or len(operands) < 2: + raise StateExpressionError(f"状态表达式运算无效: {kind or 'empty'}") + if not all(isinstance(item, Mapping) for item in operands): + raise StateExpressionError(f"{kind} 的操作数必须是表达式") + operand_types = [ + _inspect_expression( + item, + variable_types, + allow_llm=allow_llm, + depth=depth + 1, + parameters=parameters, + ) + for item in operands + ] + if kind in {"and", "or"}: + if any(item != "boolean" for item in operand_types): + raise StateExpressionError(f"{kind} 只能组合布尔值") + return "boolean" + if kind in {"eq", "neq"}: + if len(set(operand_types)) != 1: + raise StateExpressionError(f"{kind} 的操作数类型必须一致") + return "boolean" + if kind in {"gt", "gte", "lt", "lte"}: + if any(item != "number" for item in operand_types): + raise StateExpressionError(f"{kind} 只能比较数字") + return "boolean" + if kind == "add" and len(set(operand_types)) == 1: + result_type = operand_types[0] + if result_type in {"number", "string"}: + return result_type + if any(item != "number" for item in operand_types): + raise StateExpressionError(f"{kind} 只能用于数字") + return "number" + + +def validate_update_rules( + rules: list[Mapping[str, Any]] | None, + definitions: Mapping[str, Mapping[str, Any]] | None, + *, + allow_llm: bool, +) -> dict[str, dict[str, str]]: + """Validate rules and return the LLM-visible JSON schema properties.""" + + rules = rules or [] + if len(rules) > MAX_UPDATE_RULES: + raise StateExpressionError( + f"状态更新规则不能超过 {MAX_UPDATE_RULES} 条" + ) + variable_types = _definition_types(definitions) + parameters: dict[str, LlmParameter] = {} + targets: set[str] = set() + llm_target_parameters: dict[str, str] = {} + for rule in rules: + target = str(_field(rule, "variable_name", "variableName") or "") + expression = rule.get("expression") + if target not in variable_types: + raise StateExpressionError(f"状态更新目标未声明: {target}") + if target in targets: + raise StateExpressionError(f"状态更新目标不能重复: {target}") + targets.add(target) + if not isinstance(expression, Mapping): + raise StateExpressionError(f"状态更新 {target} 缺少表达式") + result_type = _inspect_expression( + expression, + variable_types, + allow_llm=allow_llm, + depth=1, + parameters=parameters, + ) + if result_type != variable_types[target]: + raise StateExpressionError( + f"状态更新 {target} 的结果类型应为 {variable_types[target]}" + ) + if allow_llm: + llm_names: set[str] = set() + _collect_llm_names(expression, llm_names) + for parameter_name in llm_names: + previous_target = llm_target_parameters.get(parameter_name) + if previous_target is not None and previous_target != target: + raise StateExpressionError( + f"LLM 参数 {parameter_name} 不能同时更新多个变量" + ) + llm_target_parameters[parameter_name] = target + return { + parameter.name: { + "type": parameter.value_type, + "description": parameter.description, + } + for parameter in parameters.values() + } + + +def _collect_llm_names(expression: Mapping[str, Any], result: set[str]) -> None: + kind = str(expression.get("type") or "") + if kind == "llm": + result.add( + str(_field(expression, "parameter_name", "parameterName") or "") + ) + return + if kind == "if": + children = [ + expression.get("condition"), + expression.get("then"), + expression.get("otherwise"), + ] + else: + children = expression.get("operands") or [] + for child in children: + if isinstance(child, Mapping): + _collect_llm_names(child, result) + + +def _evaluate( + expression: Mapping[str, Any], + snapshot: Mapping[str, Any], + arguments: Mapping[str, Any], +) -> Any: + kind = str(expression.get("type") or "") + if kind == "constant": + return expression.get("value") + if kind == "variable": + name = str(_field(expression, "variable_name", "variableName") or "") + if name not in snapshot: + raise StateExpressionError(f"状态变量尚未赋值: {name}") + return snapshot[name] + if kind == "llm": + name = str(_field(expression, "parameter_name", "parameterName") or "") + if name not in arguments: + raise StateExpressionError(f"缺少 LLM 提取参数: {name}") + return arguments[name] + if kind == "if": + branch = "then" if bool( + _evaluate(expression["condition"], snapshot, arguments) + ) else "otherwise" + return _evaluate(expression[branch], snapshot, arguments) + + values = [ + _evaluate(item, snapshot, arguments) + for item in expression.get("operands") or [] + ] + if kind == "and": + return all(values) + if kind == "or": + return any(values) + if kind == "eq": + return all(value == values[0] for value in values[1:]) + if kind == "neq": + return all(value != values[0] for value in values[1:]) + if kind == "gt": + return all(left > right for left, right in zip(values, values[1:])) + if kind == "gte": + return all(left >= right for left, right in zip(values, values[1:])) + if kind == "lt": + return all(left < right for left, right in zip(values, values[1:])) + if kind == "lte": + return all(left <= right for left, right in zip(values, values[1:])) + if kind == "add": + result = values[0] + for value in values[1:]: + result += value + return result + if kind == "subtract": + result = values[0] + for value in values[1:]: + result -= value + return result + if kind == "multiply": + result = values[0] + for value in values[1:]: + result *= value + return result + if kind == "divide": + result = values[0] + for value in values[1:]: + if value == 0: + raise StateExpressionError("状态表达式不能除以零") + result /= value + return result + raise StateExpressionError(f"不支持的状态表达式: {kind}") + + +def evaluate_update_rules( + rules: list[Mapping[str, Any]] | None, + snapshot: Mapping[str, Any], + arguments: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Evaluate every rule from the same snapshot without mutating state.""" + + return { + str(_field(rule, "variable_name", "variableName")): _evaluate( + rule["expression"], snapshot, arguments or {} + ) + for rule in rules or [] + } diff --git a/backend/services/system_tools.py b/backend/services/system_tools.py index f43ceed..580c952 100644 --- a/backend/services/system_tools.py +++ b/backend/services/system_tools.py @@ -1,56 +1,45 @@ -"""Shared contracts for platform-owned conversation system tools.""" +"""Contracts and adapters for platform-owned built-in conversation tools.""" from __future__ import annotations -from collections.abc import Iterable, Mapping -from typing import Any +from typing import Any, Mapping + +from models import RuntimeTool +from services.state_expressions import validate_update_rules SYSTEM_TOOL_SPECS: dict[str, dict[str, str]] = { "end_conversation": { - "id": "tool_end_call_default", "name": "结束对话", "function_name": "end_conversation", - "description": ( - "礼貌地结束本次对话。当用户明确告别、表示任务已完成" - "或要求挂断时调用。" - ), + "description": "礼貌地结束本次对话。", }, "update_state": { - "id": "tool_update_state_default", "name": "更新状态", "function_name": "update_state", - "description": ( - "静默更新本次对话中已经声明并明确授权的动态变量。" - "只提交本轮获得或确认的信息，更新后继续当前回答。" - ), + "description": "静默更新已经声明并明确授权的会话状态。", }, "skip_turn": { - "id": "tool_skip_turn_default", "name": "跳过本轮", "function_name": "skip_turn", - "description": ( - "跳过当前轮次，不生成任何语音回复。" - "仅当用户明确要求稍等、话还没说完，或输入可确认只是噪音时调用。" - ), + "description": "跳过当前轮次，不生成语音回复。", }, "request_human_handoff": { - "id": "tool_request_human_handoff_default", "name": "转接人工", "function_name": "request_human_handoff", - "description": ( - "提交人工接管请求。当用户明确要求人工服务、投诉升级或 AI 无法" - "解决时调用。该工具只提交请求，不代表人工已经接通；调用后继续" - "回复用户并说明正在等待人工响应。" - ), + "description": "提交人工接管请求，并继续告知用户当前进展。", }, } SYSTEM_TOOL_KINDS = frozenset(SYSTEM_TOOL_SPECS) +RESERVED_TOOL_NAMES = frozenset( + spec["function_name"] for spec in SYSTEM_TOOL_SPECS.values() +) def system_tool_kind(definition: Mapping[str, Any] | None) -> str | None: - """Return the supported platform behavior declared by a System resource.""" + """Compatibility helper for old fixtures and the one-time migration.""" + config = (definition or {}).get("config") if not isinstance(config, Mapping): return None @@ -58,28 +47,56 @@ def system_tool_kind(definition: Mapping[str, Any] | None) -> str | None: return kind if kind in SYSTEM_TOOL_KINDS else None +def runtime_system_tools( + built_in_tools: Mapping[str, Any] | None, +) -> list[RuntimeTool]: + """Adapt immutable built-in configs to existing brain tool builders.""" + + result: list[RuntimeTool] = [] + aliases = { + "endConversation": "end_conversation", + "updateState": "update_state", + "skipTurn": "skip_turn", + "requestHumanHandoff": "request_human_handoff", + } + config_aliases = { + "messageType": "message_type", + "customMessage": "custom_message", + "captureReason": "capture_reason", + "allowEndInterruption": "allow_end_interruption", + } + for raw_kind, config in (built_in_tools or {}).items(): + kind = aliases.get(str(raw_kind), str(raw_kind)) + if kind not in SYSTEM_TOOL_SPECS or not isinstance(config, Mapping): + continue + spec = SYSTEM_TOOL_SPECS[kind] + normalized_config = dict(config) + for camel, snake in config_aliases.items(): + if camel in normalized_config and snake not in normalized_config: + normalized_config[snake] = normalized_config.pop(camel) + result.append( + RuntimeTool( + id=f"builtin_{kind}", + name=spec["name"], + function_name=spec["function_name"], + type="system", + description=str( + normalized_config.get("description") or spec["description"] + ), + definition={ + "type": "system", + "config": {"kind": kind, **normalized_config}, + }, + ) + ) + return result + + def state_update_properties( definitions: Mapping[str, Mapping[str, Any]] | None, *, - allowed_names: Iterable[str] | None = None, + rules: list[Mapping[str, Any]] | None = None, ) -> dict[str, dict[str, Any]]: - """Build an explicit LLM schema for writable declared variables.""" - definitions = definitions or {} - names = ( - list(dict.fromkeys(str(name) for name in allowed_names)) - if allowed_names is not None - else list(definitions) - ) - properties: dict[str, dict[str, Any]] = {} - for name in names: - definition = definitions.get(name) - if not isinstance(definition, Mapping): - continue - variable_type = str(definition.get("type") or "string") - if variable_type not in {"string", "number", "boolean"}: - continue - properties[name] = { - "type": variable_type, - "description": f"更新动态变量 {name}。", - } - return properties + """Build the LLM schema from explicit v4 expression rules.""" + + return validate_update_rules(rules or [], definitions, allow_llm=True) diff --git a/backend/services/test_runs/mock_tools.py b/backend/services/test_runs/mock_tools.py index 7b1d330..c20911d 100644 --- a/backend/services/test_runs/mock_tools.py +++ b/backend/services/test_runs/mock_tools.py @@ -6,7 +6,7 @@ import asyncio import json from typing import Any -from models import RuntimeTool +from models import RuntimeTool, RuntimeToolBinding from services.runtime_variables import DynamicVariableStore from services.tool_executor import ToolExecutionError, ToolExecutor from test_schemas import ToolCallExpectedBehavior @@ -34,9 +34,9 @@ class MockToolExecutor(ToolExecutor): tool: RuntimeTool, arguments: dict[str, Any] | None = None, *, + binding: RuntimeToolBinding | None = None, result_assignments: dict[str, str] | None = None, ) -> dict[str, Any]: - self.register_secrets(tool) behavior = self._mocks_by_tool_id.get(tool.id) or self._mocks_by_function.get( tool.function_name ) @@ -62,7 +62,11 @@ class MockToolExecutor(ToolExecutor): return self._apply_result_assignments( tool, result, - result_assignments=result_assignments, + result_assignments=( + result_assignments + if result_assignments is not None + else (binding.result_assignments if binding is not None else {}) + ), ) @@ -72,6 +76,9 @@ class TextClientToolPort: def __init__(self) -> None: self.auto_confirmed_messages: list[dict[str, Any]] = [] + def supports(self, function_name: str) -> bool: + return function_name == "show_message" + async def call( self, function_name: str, diff --git a/backend/services/tool_executor.py b/backend/services/tool_executor.py index f3c6725..02fa48a 100644 --- a/backend/services/tool_executor.py +++ b/backend/services/tool_executor.py @@ -7,7 +7,7 @@ from typing import Any from urllib.parse import quote import httpx -from models import RuntimeTool +from models import RuntimeTool, RuntimeToolBinding from services.runtime_variables import ( DynamicVariableError, @@ -40,15 +40,8 @@ class ToolExecutor: def set_client_tools(self, client_tools: ClientToolPort | None) -> None: self._client_tools = client_tools - def register_secrets(self, tool: RuntimeTool) -> None: - dynamic = (tool.secrets or {}).get("dynamic_variables") or {} - for name, value in dynamic.items(): - if not str(name).startswith("secret__"): - raise DynamicVariableError(f"工具密钥变量必须以 secret__ 开头: {name}") - self.store.secrets[str(name)] = str(value) - @staticmethod - def schema_parts(tool: RuntimeTool) -> tuple[dict[str, Any], list[str]]: + def input_schema(tool: RuntimeTool) -> tuple[dict[str, Any], list[str]]: config = (tool.definition or {}).get("config") or {} if tool.type == "mcp": schema = config.get("input_schema") or {} @@ -72,19 +65,58 @@ class ToolExecutor: ] return properties, required + @classmethod + def schema_parts( + cls, + tool: RuntimeTool, + binding: RuntimeToolBinding | None = None, + ) -> tuple[dict[str, Any], list[str]]: + properties, required = cls.input_schema(tool) + if binding is None: + return properties, required + visible = { + name + for name, source in binding.argument_bindings.items() + if str((source or {}).get("source") or "") == "model" + } + missing_required = [ + name for name in required if name not in binding.argument_bindings + ] + if missing_required: + raise ToolExecutionError( + "缺少必填工具参数绑定: " + ", ".join(missing_required) + ) + return ( + {name: spec for name, spec in properties.items() if name in visible}, + [name for name in required if name in visible], + ) + async def execute( self, tool: RuntimeTool, arguments: dict[str, Any] | None = None, *, + binding: RuntimeToolBinding | None = None, result_assignments: dict[str, str] | None = None, ) -> dict[str, Any]: - self.register_secrets(tool) - normalized_arguments = dict(arguments or {}) + normalized_arguments = self.resolve_arguments( + tool, + binding, + dict(arguments or {}), + ) + template_values = self.resolve_template_bindings(binding) if tool.type == "http": - result = await self._execute_http(tool, normalized_arguments) + result = await self._execute_http( + tool, + normalized_arguments, + template_values=template_values, + ) elif tool.type == "mcp": - result = await self._execute_mcp(tool, normalized_arguments) + result = await self._execute_mcp( + tool, + normalized_arguments, + template_values=template_values, + ) elif tool.type == "client": result = await self._execute_client(tool, normalized_arguments) else: @@ -94,26 +126,111 @@ class ToolExecutor: return self._apply_result_assignments( tool, result, - result_assignments=result_assignments, + result_assignments=( + result_assignments + if result_assignments is not None + else (binding.result_assignments if binding is not None else {}) + ), ) + def resolve_arguments( + self, + tool: RuntimeTool, + binding: RuntimeToolBinding | None, + model_arguments: dict[str, Any], + ) -> dict[str, Any]: + if binding is None: + return model_arguments + properties, required = self.input_schema(tool) + visible_model_parameters = { + name + for name, source in binding.argument_bindings.items() + if str((source or {}).get("source") or "") == "model" + } + resolved: dict[str, Any] = {} + for name, source in binding.argument_bindings.items(): + if name not in properties: + raise ToolExecutionError(f"工具参数不存在: {name}") + resolved[name] = self._resolve_argument_source( + name, + source, + model_arguments, + ) + unknown_model_arguments = sorted( + set(model_arguments) - visible_model_parameters + ) + if unknown_model_arguments: + raise ToolExecutionError( + "模型提供了未授权工具参数: " + + ", ".join(unknown_model_arguments) + ) + missing = [name for name in required if name not in resolved] + if missing: + raise ToolExecutionError("缺少必填工具参数: " + ", ".join(missing)) + return resolved + + def resolve_template_bindings( + self, + binding: RuntimeToolBinding | None, + ) -> dict[str, Any]: + if binding is None: + return {} + return { + name: self._resolve_source(source, {}) + for name, source in binding.template_bindings.items() + } + + def _resolve_source( + self, + source: dict[str, Any], + model_arguments: dict[str, Any], + ) -> Any: + kind = str((source or {}).get("source") or "") + if kind == "model": + raise ToolExecutionError("模型参数来源缺少参数上下文") + if kind == "variable": + name = str( + source.get("variable_name") or source.get("variableName") or "" + ) + return self.store.value(name) + if kind == "literal": + return deepcopy(source.get("value")) + if kind == "template": + return self.store.render_data(deepcopy(source.get("value"))) + raise ToolExecutionError(f"不支持的参数来源: {kind or 'empty'}") + + def _resolve_argument_source( + self, + name: str, + source: dict[str, Any], + model_arguments: dict[str, Any], + ) -> Any: + if str((source or {}).get("source") or "") == "model": + if name not in model_arguments: + raise ToolExecutionError(f"模型未提供工具参数: {name}") + return model_arguments[name] + return self._resolve_source(source, model_arguments) + async def _execute_http( self, tool: RuntimeTool, arguments: dict[str, Any], + *, + template_values: dict[str, Any], ) -> dict[str, Any]: config = (tool.definition or {}).get("config") or {} parameters = list(config.get("parameters") or []) - url = self.store.render(str(config.get("url") or "")) - configured_headers = self.store.render_data( + renderer = self._tool_renderer(tool, template_values) + url = renderer.render(str(config.get("url") or "")) + configured_headers = renderer.render_data( deepcopy(config.get("headers") or {}), allow_secrets=True ) - secret_headers = self.store.render_data( + secret_headers = renderer.render_data( deepcopy((tool.secrets or {}).get("headers") or {}), allow_secrets=True ) headers: dict[str, str] = {} query: dict[str, object] = {} - body = self.store.render_data(deepcopy(config.get("body") or {})) + body = renderer.render_data(deepcopy(config.get("body") or {})) for parameter in parameters: name = str(parameter.get("name") or "") @@ -169,6 +286,8 @@ class ToolExecutor: self, tool: RuntimeTool, arguments: dict[str, Any], + *, + template_values: dict[str, Any], ) -> dict[str, Any]: server = tool.mcp_server if server is None: @@ -178,14 +297,12 @@ class ToolExecutor: if not remote_tool_name: raise ToolExecutionError("MCP 工具缺少远端工具名称") + renderer = self._tool_renderer(tool, template_values) resolved_server = server.model_copy( update={ - "url": self.store.render(server.url), - "headers": self.store.render_data(server.headers), - "secret_headers": self.store.render_data( - server.secret_headers, - allow_secrets=True, - ), + "url": renderer.render(server.url), + "headers": renderer.render_data(server.headers), + "secret_headers": dict(server.secret_headers), } ) try: @@ -198,6 +315,33 @@ class ToolExecutor: raise ToolExecutionError(str(exc)) from exc return {"status": "ok", "data": payload} + def _tool_renderer( + self, + tool: RuntimeTool, + template_values: dict[str, Any], + ) -> DynamicVariableStore: + values = dict(self.store.values) + values.update( + {f"input__{name}": value for name, value in template_values.items()} + ) + secrets = { + str(name): str(value) + for name, value in ( + (tool.secrets or {}).get("dynamic_variables") or {} + ).items() + } + invalid = [name for name in secrets if not name.startswith("secret__")] + if invalid: + raise DynamicVariableError( + "工具密钥变量必须以 secret__ 开头: " + ", ".join(invalid) + ) + return DynamicVariableStore( + values, + secrets, + optional_names=self.store.optional_names, + variable_types=self.store.variable_types, + ) + async def _execute_client( self, tool: RuntimeTool, @@ -205,6 +349,9 @@ class ToolExecutor: ) -> dict[str, Any]: if self._client_tools is None: raise ToolExecutionError("当前运行模式不支持客户端工具") + supports = getattr(self._client_tools, "supports", None) + if supports is not None and not supports(tool.function_name): + raise ToolExecutionError(f"客户端未注册工具: {tool.function_name}") config = (tool.definition or {}).get("config") or {} policy = policy_for_tool(tool) try: @@ -225,14 +372,9 @@ class ToolExecutor: *, result_assignments: dict[str, str] | None, ) -> dict[str, Any]: - config = (tool.definition or {}).get("config") or {} - assignments = ( - result_assignments - if result_assignments is not None - else config.get("dynamic_variable_assignments") or {} - ) + assignments = result_assignments or {} payload = result.get("data") - updated: list[str] = [] + staged: dict[str, Any] = {} for variable_name, path in assignments.items(): try: value = value_at_path(payload, str(path)) @@ -241,6 +383,6 @@ class ToolExecutor: value = value_at_path({"response": payload}, str(path)) except KeyError: continue - self.store.assign(str(variable_name), value) - updated.append(str(variable_name)) + staged[str(variable_name)] = value + updated = self.store.assign_declared_many(staged) if staged else [] return {**result, "updated_variables": updated} diff --git a/backend/services/workflow/realtime.py b/backend/services/workflow/realtime.py index 08267e7..a103417 100644 --- a/backend/services/workflow/realtime.py +++ b/backend/services/workflow/realtime.py @@ -8,7 +8,7 @@ from dataclasses import dataclass from typing import Any from loguru import logger -from models import AssistantConfig, RuntimeTool +from models import AssistantConfig, RuntimeTool, RuntimeToolBinding from pipecat.frames.frames import OutputTransportMessageUrgentFrame from pipecat.utils.time import time_now_iso8601 @@ -27,7 +27,8 @@ from services.message_stage import ( from services.pipecat.call_lifecycle import playback_marker_for from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult from services.runtime_variables import DynamicVariableError, DynamicVariableStore -from services.system_tools import state_update_properties, system_tool_kind +from services.system_tools import runtime_system_tools, state_update_properties, system_tool_kind +from services.state_expressions import evaluate_update_rules from services.tool_executor import ToolExecutionError, ToolExecutor from services.tool_policy import policy_for_tool from services.vision import ( @@ -434,27 +435,43 @@ class WorkflowRealtimeController: tools.append(tool) handlers[tool.name] = handler - for tool_id in stage.tool_ids: - runtime_tool = self._tool_by_id.get(str(tool_id)) + for raw_binding in stage.tool_bindings: + binding = RuntimeToolBinding(**raw_binding) + runtime_tool = self._tool_by_id.get(binding.tool_id) if not runtime_tool: continue - if runtime_tool.type == "system": - built = self._system_tool( - runtime_tool, - node_id=node_id, - transition_id=transition_id, - state_variable_names=stage.state_variable_names, + if ( + runtime_tool.type == "client" + and self._runtime.client_tools is not None + and getattr( + self._runtime.client_tools, + "supports", + lambda _name: True, + )(runtime_tool.function_name) is False + ): + logger.warning( + f"跳过客户端未上报的工具: {runtime_tool.function_name}" ) - elif runtime_tool.type in {"http", "mcp", "client"}: + continue + if runtime_tool.type in {"http", "mcp", "client"}: built = self._business_tool( runtime_tool, node_id=node_id, transition_id=transition_id, + binding=binding, ) else: built = None if built: add(*built) + for runtime_tool in runtime_system_tools(stage.built_in_tools): + built = self._system_tool( + runtime_tool, + node_id=node_id, + transition_id=transition_id, + ) + if built: + add(*built) knowledge = self._knowledge_tool(node_id, transition_id) if knowledge: @@ -478,7 +495,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "当前 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) activation = await self._follow_edge( edge, @@ -489,7 +506,7 @@ class WorkflowRealtimeController: "status": "success", "targetNodeId": str(edge.get("target") or ""), }, - continue_response=activation.continue_response, + follow_up=("continue" if activation.continue_response else "none"), after_output=activation.after_output, ) @@ -507,14 +524,18 @@ class WorkflowRealtimeController: *, node_id: str, transition_id: int, + binding: RuntimeToolBinding, ) -> tuple[RealtimeTool, ToolHandler]: - properties, required = self._tools.schema_parts(tool) - self._tools.register_secrets(tool) + properties, required = self._tools.schema_parts(tool, binding) policy = policy_for_tool(tool) async def handler(arguments: dict[str, Any]) -> RealtimeToolResult: try: - result = await self._tools.execute(tool, arguments) + result = await self._tools.execute( + tool, + arguments, + binding=binding, + ) except ToolExecutionError as exc: return RealtimeToolResult( {"status": "error", "message": str(exc)} @@ -527,7 +548,7 @@ class WorkflowRealtimeController: "status": "stale", "message": "工具完成时当前 Agent 已经切换。", }, - continue_response=False, + follow_up="none", ) activation = await self._after_variables_changed( list(result.get("updated_variables") or []), @@ -535,10 +556,14 @@ class WorkflowRealtimeController: ) return RealtimeToolResult( result, - continue_response=( - activation.continue_response - if activation - else policy.runs_llm_after_result + follow_up=( + "continue" + if ( + activation.continue_response + if activation + else policy.runs_llm_after_result + ) + else "none" ), after_output=activation.after_output if activation else None, ) @@ -575,7 +600,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "当前 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) try: question = resolve_vision_question( @@ -609,7 +634,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "获取画面时 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) try: @@ -632,7 +657,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "分析画面时 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) return RealtimeToolResult( { @@ -662,25 +687,21 @@ class WorkflowRealtimeController: *, node_id: str, transition_id: int, - state_variable_names: tuple[str, ...], ) -> tuple[RealtimeTool, ToolHandler] | None: kind = system_tool_kind(tool.definition or {}) if kind == "update_state": - allowed = frozenset(state_variable_names) + config = (tool.definition or {}).get("config") or {} + rules = list(config.get("updates") or []) async def update_state(arguments: dict[str, Any]) -> RealtimeToolResult: - unauthorized = sorted(set(arguments) - allowed) - if unauthorized: - return RealtimeToolResult( - { - "status": "error", - "message": "状态变量未获当前节点授权:" - + ",".join(unauthorized), - } - ) try: - changed = self._store.assign_declared_many(arguments) - except DynamicVariableError as exc: + values = evaluate_update_rules( + rules, + dict(self._store.values), + arguments, + ) + changed = self._store.assign_declared_many(values) + except (DynamicVariableError, ValueError) as exc: return RealtimeToolResult( {"status": "error", "message": f"状态更新失败:{exc}"} ) @@ -688,7 +709,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "当前 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) activation = await self._after_variables_changed( changed, @@ -700,8 +721,10 @@ class WorkflowRealtimeController: "changed": changed, "variables": self._store.public_values(), }, - continue_response=( - activation.continue_response if activation else True + follow_up=( + "continue" + if (activation.continue_response if activation else True) + else "none" ), after_output=activation.after_output if activation else None, ) @@ -712,7 +735,7 @@ class WorkflowRealtimeController: description=tool.description or "更新已授权的动态变量。", properties=state_update_properties( self._cfg.dynamic_variable_definitions, - allowed_names=state_variable_names, + rules=rules, ), ), update_state, @@ -725,7 +748,7 @@ class WorkflowRealtimeController: "action": "skip_turn", "reason": str(arguments.get("reason") or ""), }, - continue_response=False, + follow_up="suppress", ) return ( @@ -813,7 +836,7 @@ class WorkflowRealtimeController: if not self._is_current(node_id, transition_id): return RealtimeToolResult( {"status": "stale", "message": "当前 Agent 已经切换。"}, - continue_response=False, + follow_up="none", ) query = str(arguments.get("query") or "").strip() if not query: @@ -932,15 +955,15 @@ class WorkflowRealtimeController: if self._runtime.set_input_enabled: self._runtime.set_input_enabled(False) data = self._engine.data(node_id) - tool = self._tool_by_id.get(str(data.get("toolId") or "")) + binding = RuntimeToolBinding(**(data.get("toolBinding") or {})) + tool = self._tool_by_id.get(binding.tool_id) result = await self._action_stages.run( ActionStageSpec( actions=( StageAction( id=node_id, tool=tool, - arguments=data.get("arguments") or {}, - result_assignments=self._action_result_assignments(data), + binding=binding, ), ), input_policy="block", @@ -961,12 +984,9 @@ class WorkflowRealtimeController: async def _enter_update_state(self, node_id: str) -> None: self._state.enter(node_id, WorkflowStatus.RUNNING_ACTION) await self._emit_active(node_id) - rendered = self._store.render_data( - self._engine.data(node_id).get("assignments") or {} - ) - if not isinstance(rendered, dict): - raise DynamicVariableError("Update State 节点赋值必须是对象") - changed = self._store.assign_declared_many(rendered) + updates = self._engine.data(node_id).get("updates") or [] + values = evaluate_update_rules(updates, dict(self._store.values)) + changed = self._store.assign_declared_many(values) if changed: await self._emit_variables("update_state", node_id, changed) @@ -1058,18 +1078,6 @@ class WorkflowRealtimeController: self._engine.data(node_id).get("completionPolicy") or "playback" ) - @staticmethod - def _action_result_assignments( - data: dict[str, Any], - ) -> dict[str, str] | None: - mode = str(data.get("resultAssignmentMode") or "none") - if mode == "inherit": - return None - if mode == "override": - value = data.get("resultAssignments") - return dict(value) if isinstance(value, dict) else {} - return {} - def _set_last_action(self, outcome) -> None: self._store.values.update( { diff --git a/backend/services/workflow_engine.py b/backend/services/workflow_engine.py index 261dbf9..195e3e8 100644 --- a/backend/services/workflow_engine.py +++ b/backend/services/workflow_engine.py @@ -1,4 +1,4 @@ -"""Pure Workflow v3 graph queries and deterministic edge evaluation.""" +"""Pure Workflow v4 graph queries and deterministic edge evaluation.""" from __future__ import annotations @@ -25,8 +25,8 @@ class AgentStageConfig: vision_model_resource_id: str | None vision_prompt_mode: str vision_prompt: str - tool_ids: tuple[str, ...] - state_variable_names: tuple[str, ...] + tool_bindings: tuple[dict[str, Any], ...] + built_in_tools: dict[str, Any] knowledge_base_id: str | None knowledge_mode: str knowledge_top_n: int @@ -155,16 +155,12 @@ class WorkflowEngine: ), vision_prompt_mode=str(source.get("visionPromptMode") or "runtime"), vision_prompt=str(source.get("visionPrompt") or "").strip(), - tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []), - # Variable write scope always belongs to the Agent node, even when - # its selected tool resources come from Workflow defaults. - state_variable_names=tuple( - dict.fromkeys( - str(name) - for name in data.get("stateVariableNames") or [] - if str(name) - ) + tool_bindings=tuple( + dict(binding) + for binding in source.get("toolBindings") or [] + if isinstance(binding, dict) and binding.get("toolId") ), + built_in_tools=dict(source.get("builtInTools") or {}), knowledge_base_id=knowledge_base_id or None, knowledge_mode=( str(source.get("knowledgeMode") or "automatic") diff --git a/backend/tests/test_brains.py b/backend/tests/test_brains.py index 7dfc435..2881049 100644 --- a/backend/tests/test_brains.py +++ b/backend/tests/test_brains.py @@ -5,7 +5,7 @@ import unittest from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch -from models import AssistantConfig, RuntimeTool +from models import AssistantConfig, RuntimeTool, RuntimeToolBinding from pipecat.flows import FlowManager from pipecat.frames.frames import ( LLMContextFrame, @@ -21,7 +21,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.aggregators.llm_context import LLMContext from pipecat.processors.frame_processor import FrameDirection -from schemas import AssistantUpsert, REALTIME_CAPABLE_TYPES, SystemToolConfig +from schemas import AssistantUpsert, BuiltInTools, REALTIME_CAPABLE_TYPES from services.brains import BrainRuntime, SPECS, build_brain from services.brains.base import GREETING_CONTEXT_MARKER from services.brains.dify_llm import ( @@ -130,6 +130,21 @@ def system_runtime_tool(kind: str, **config) -> RuntimeTool: ) +def built_in_config(kind: str, **config) -> dict: + key = { + "end_conversation": "end_conversation", + "update_state": "update_state", + "skip_turn": "skip_turn", + "request_human_handoff": "request_human_handoff", + }[kind] + base = {"description": f"{kind} system tool", **config} + return {key: base} + + +def merge_builtins(*configs: dict) -> dict: + return {key: value for config in configs for key, value in config.items()} + + class BrainRegistryTests(unittest.TestCase): def test_capability_matrix(self): self.assertEqual( @@ -139,7 +154,7 @@ class BrainRegistryTests(unittest.TestCase): }, { "prompt": frozenset({"pipeline", "realtime"}), - "workflow": frozenset({"pipeline"}), + "workflow": frozenset({"pipeline", "realtime"}), "dify": frozenset({"pipeline"}), "fastgpt": frozenset({"pipeline"}), }, @@ -157,13 +172,13 @@ class BrainRegistryTests(unittest.TestCase): with self.assertRaisesRegex(ValueError, "尚未实现"): build_brain(AssistantConfig(type="opencode")) - def test_workflow_realtime_is_rejected_at_schema_boundary(self): - with self.assertRaises(ValueError): - AssistantUpsert( - name="workflow", - type="workflow", - runtimeMode="realtime", - ) + def test_workflow_realtime_is_accepted_at_schema_boundary(self): + assistant = AssistantUpsert( + name="workflow", + type="workflow", + runtimeMode="realtime", + ) + self.assertEqual(assistant.runtime_mode, "realtime") def test_prompt_realtime_keeps_dynamic_variable_definitions(self): assistant = AssistantUpsert( @@ -180,36 +195,27 @@ class BrainRegistryTests(unittest.TestCase): ) self.assertIn("user_name", assistant.dynamic_variable_definitions) - def test_system_tool_config_accepts_all_platform_actions(self): - for kind in ( - "end_conversation", - "update_state", - "skip_turn", - "request_human_handoff", - ): - self.assertEqual(SystemToolConfig(kind=kind).kind, kind) - - def test_end_interruption_is_opt_in(self): - self.assertFalse(SystemToolConfig().allow_end_interruption) - self.assertTrue( - SystemToolConfig(allowEndInterruption=True).allow_end_interruption + def test_built_in_tools_are_typed_platform_config(self): + tools = BuiltInTools( + endConversation={"allowEndInterruption": True}, + updateState={"updates": []}, + skipTurn={}, + requestHumanHandoff={}, ) - - def test_system_tool_config_rejects_unknown_kind(self): - with self.assertRaises(ValueError): - SystemToolConfig(kind="magic") + self.assertTrue(tools.end_conversation.allow_end_interruption) + self.assertEqual(tools.update_state.updates, []) def test_workflow_keeps_dynamic_variables_and_tool_bindings(self): assistant = AssistantUpsert( name="workflow", type="workflow", - toolIds=["tool_a"], + toolBindings=[{"toolId": "tool_a"}], dynamicVariableDefinitions={ "customer": {"type": "string", "required": False, "default": "王先生"} }, graph={}, ) - self.assertEqual(assistant.tool_ids, ["tool_a"]) + self.assertEqual(assistant.tool_bindings, []) self.assertIn("customer", assistant.dynamic_variable_definitions) @@ -310,13 +316,16 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): cfg = AssistantConfig( type="prompt", tools=[startup_tool, conversation_tool], - llm_tool_ids=[conversation_tool.id], + tool_bindings=[RuntimeToolBinding(tool_id=conversation_tool.id)], startup={ "actions": [ { "id": "opening_message", "phase": "opening", - "tool_id": startup_tool.id, + "toolBinding": { + "toolId": startup_tool.id, + "argumentBindings": {}, + }, "required": True, } ] @@ -361,7 +370,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): { "id": tool.id, "phase": "preflight", - "tool_id": tool.id, + "toolBinding": { + "toolId": tool.id, + "argumentBindings": {}, + }, "required": True, } for tool in tools @@ -422,8 +434,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): { "id": "opening_data", "phase": "opening", - "tool_id": "opening_data", - "arguments": {}, + "toolBinding": { + "toolId": "opening_data", + "argumentBindings": {}, + }, "required": True, } ], @@ -848,26 +862,13 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(queued_frames, []) async def test_end_conversation_system_tool_is_owned_by_prompt_brain(self): - brain = build_brain( - AssistantConfig( - type="prompt", - tools=[ - RuntimeTool( - id="end-call", - name="结束通话", - function_name="end_call", - type="system", - definition={ - "config": { - "kind": "end_conversation", - "message_type": "none", - "capture_reason": True, - } - }, - ) - ], - ) + cfg = AssistantConfig( + type="prompt", + built_in_tools=built_in_config( + "end_conversation", capture_reason=True + ), ) + brain = build_brain(cfg) llm = FakeLLM() call_end = FakeCallEnd() visible_tools = [] @@ -876,23 +877,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): pass await brain.setup( - AssistantConfig( - type="prompt", - tools=[ - RuntimeTool( - id="end-call", - name="结束通话", - function_name="end_call", - type="system", - definition={ - "config": { - "kind": "end_conversation", - "capture_reason": True, - } - }, - ) - ], - ), + cfg, BrainRuntime( context=LLMContext(messages=[]), llm=llm, @@ -903,10 +888,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): ), ) - self.assertEqual(visible_tools[0].name, "end_call") + self.assertEqual(visible_tools[0].name, "end_conversation") params = FakeFunctionParams({"reason": "用户已完成咨询"}) await brain.on_assistant_text_start("closing-turn") - await llm.functions["end_call"](params) + await llm.functions["end_conversation"](params) self.assertEqual(call_end.reason, "用户已完成咨询") self.assertFalse(call_end.finished) await brain.on_assistant_text_end("closing-turn", "", False) @@ -916,19 +901,10 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(params.result["action"], "ending_call") async def test_system_end_conversation_waits_for_generated_closing_speech(self): - tool = RuntimeTool( - id="end-call", - name="结束通话", - function_name="end_call", - type="system", - definition={ - "config": { - "kind": "end_conversation", - "message_type": "none", - } - }, + cfg = AssistantConfig( + type="prompt", + built_in_tools=built_in_config("end_conversation"), ) - cfg = AssistantConfig(type="prompt", tools=[tool]) brain = build_brain(cfg) llm = FakeLLM() call_end = FakeCallEnd() @@ -945,7 +921,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): ), ) await brain.on_assistant_text_start("closing-turn") - await llm.functions["end_call"](FakeFunctionParams({})) + await llm.functions["end_conversation"](FakeFunctionParams({})) await brain.on_assistant_text_end( "closing-turn", "感谢您的来电。祝您生活愉快，再见。", @@ -966,12 +942,23 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): "default": None, } }, - tools=[ - system_runtime_tool("end_conversation"), - system_runtime_tool("update_state"), - system_runtime_tool("skip_turn"), - system_runtime_tool("request_human_handoff"), - ], + built_in_tools=merge_builtins( + built_in_config("end_conversation"), + built_in_config( + "update_state", + updates=[{ + "variableName": "user_name", + "expression": { + "type": "llm", + "parameterName": "user_name", + "valueType": "string", + "description": "更新动态变量 user_name。", + }, + }], + ), + built_in_config("skip_turn"), + built_in_config("request_human_handoff"), + ), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1021,7 +1008,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): async def test_end_conversation_ends_call_after_generated_speech(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("end_conversation")], + built_in_tools=built_in_config("end_conversation"), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1059,12 +1046,9 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): async def test_end_conversation_passes_interruptible_tool_setting(self): cfg = AssistantConfig( type="prompt", - tools=[ - system_runtime_tool( - "end_conversation", - allow_end_interruption=True, - ) - ], + built_in_tools=built_in_config( + "end_conversation", allow_end_interruption=True + ), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1089,7 +1073,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): async def test_end_conversation_finishes_when_no_speech(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("end_conversation")], + built_in_tools=built_in_config("end_conversation"), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1119,7 +1103,18 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): dynamic_variable_definitions={ "user_name": {"type": "string", "required": False, "default": None} }, - tools=[system_runtime_tool("update_state")], + built_in_tools=built_in_config( + "update_state", + updates=[{ + "variableName": "user_name", + "expression": { + "type": "llm", + "parameterName": "user_name", + "valueType": "string", + "description": "更新用户名", + }, + }], + ), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1160,10 +1155,24 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): ) self.assertTrue(prompts[-1].endswith("当前用户名: 王小明")) - async def test_update_state_rejects_undeclared_variable(self): + async def test_update_state_rejects_missing_configured_value(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("update_state")], + dynamic_variable_definitions={ + "known_key": {"type": "string", "required": False, "default": None} + }, + built_in_tools=built_in_config( + "update_state", + updates=[{ + "variableName": "known_key", + "expression": { + "type": "llm", + "parameterName": "known_key", + "valueType": "string", + "description": "更新已知值", + }, + }], + ), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1183,12 +1192,12 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): params = FakeFunctionParams({"unknown_key": "x"}) await llm.functions["update_state"](params) self.assertEqual(params.result["status"], "error") - self.assertIn("未声明", params.result["message"]) + self.assertIn("缺少", params.result["message"]) async def test_skip_turn_suppresses_response(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("skip_turn")], + built_in_tools=built_in_config("skip_turn"), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1214,7 +1223,7 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): async def test_request_human_handoff_keeps_call_available(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("request_human_handoff")], + built_in_tools=built_in_config("request_human_handoff"), ) brain = build_brain(cfg) llm = FakeLLM() @@ -1320,14 +1329,23 @@ class PromptBrainTests(unittest.IsolatedAsyncioTestCase): "required": False, }, ], - "dynamic_variable_assignments": { - "order_status": "response.order.status" - }, } }, secrets={"dynamic_variables": {"secret__token": "server-token"}}, ) ], + tool_bindings=[ + RuntimeToolBinding( + tool_id="lookup", + argument_bindings={ + "order_id": {"source": "model"}, + "Authorization": {"source": "model"}, + }, + result_assignments={ + "order_status": "response.order.status" + }, + ) + ], ), {}, assistant_id="asst_1", @@ -1380,12 +1398,23 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "type": "agent", "data": { "inheritGlobalConfig": False, - "toolIds": [ - "tool_update_state", - "tool_skip_turn", - "tool_request_human_handoff", - ], - "stateVariableNames": ["customer_name"], + "toolBindings": [], + "builtInTools": merge_builtins( + built_in_config( + "update_state", + updates=[{ + "variableName": "customer_name", + "expression": { + "type": "llm", + "parameterName": "customer_name", + "valueType": "string", + "description": "更新客户姓名", + }, + }], + ), + built_in_config("skip_turn"), + built_in_config("request_human_handoff"), + ), }, }, ], @@ -1403,11 +1432,6 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "default": None, }, }, - tools=[ - system_runtime_tool("update_state"), - system_runtime_tool("skip_turn"), - system_runtime_tool("request_human_handoff"), - ], ), {}, assistant_id="asst_workflow_system_tools", @@ -1435,6 +1459,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): ) brain._emit_variables = AsyncMock() brain._refresh_agent_prompt = AsyncMock() + brain._state.enter("agent", WorkflowStatus.WAITING_USER) node_config = brain._agent_config("agent") functions = {tool.name: tool for tool in node_config["functions"]} @@ -1465,7 +1490,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): None, ) self.assertEqual(unauthorized["status"], "error") - self.assertIn("未获当前节点授权", unauthorized["message"]) + self.assertIn("缺少", unauthorized["message"]) handoff = await functions["request_human_handoff"].handler( {"reason": "用户要求人工"}, @@ -1494,10 +1519,19 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "id": "set_state", "type": "update_state", "data": { - "assignments": { - "confirmed": True, - "display_name": "{{source_name}}", - } + "updates": [ + { + "variableName": "confirmed", + "expression": {"type": "constant", "value": True}, + }, + { + "variableName": "display_name", + "expression": { + "type": "variable", + "variableName": "source_name", + }, + }, + ] }, }, ], @@ -1640,8 +1674,15 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): }, ) - self.assertEqual(brain._flow_tool(timeout_tool, "start").timeout_secs, 7.0) - self.assertIsNone(brain._flow_tool(session_tool, "start").timeout_secs) + timeout_binding = RuntimeToolBinding(tool_id=timeout_tool.id) + session_binding = RuntimeToolBinding(tool_id=session_tool.id) + self.assertEqual( + brain._flow_tool(timeout_tool, "start", timeout_binding).timeout_secs, + 7.0, + ) + self.assertIsNone( + brain._flow_tool(session_tool, "start", session_binding).timeout_secs + ) async def test_flow_transition_honors_strict_agent_entry_mode(self): brain = WorkflowBrain( @@ -1961,9 +2002,12 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "id": "lookup_action", "type": "action", "data": { - "toolId": "lookup", - "resultAssignments": { - "order_status": "order.status" + "toolBinding": { + "toolId": "lookup", + "argumentBindings": {}, + "resultAssignments": { + "order_status": "order.status" + }, }, }, }, @@ -1984,8 +2028,8 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): async def queue_frame(frame): queued.append(frame) - async def execute(_tool, _arguments, *, result_assignments=None): - self.assertEqual(result_assignments, {"order_status": "order.status"}) + async def execute(_tool, _arguments, *, binding=None, result_assignments=None): + self.assertEqual(binding.result_assignments, {"order_status": "order.status"}) brain._store.assign("order_status", "paid") return { "status": "ok", @@ -2032,7 +2076,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(completed_event["workflowRevision"], brain._engine.revision) self.assertNotIn("result", completed_event["outcome"]) - async def test_action_result_assignment_modes_reach_tool_executor(self): + async def test_action_binding_reaches_tool_executor(self): tool = RuntimeTool( id="client_action", name="客户端操作", @@ -2051,25 +2095,30 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "id": "inherit_action", "type": "action", "data": { - "toolId": "client_action", - "resultAssignmentMode": "inherit", + "toolBinding": { + "toolId": "client_action", + "resultAssignments": {}, + }, }, }, { "id": "override_action", "type": "action", "data": { - "toolId": "client_action", - "resultAssignmentMode": "override", - "resultAssignments": {"choice": "action"}, + "toolBinding": { + "toolId": "client_action", + "resultAssignments": {"choice": "action"}, + }, }, }, { "id": "none_action", "type": "action", "data": { - "toolId": "client_action", - "resultAssignmentMode": "none", + "toolBinding": { + "toolId": "client_action", + "resultAssignments": {}, + }, }, }, ], @@ -2080,8 +2129,8 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): ) captured = [] - async def execute(_tool, _arguments, *, result_assignments=None): - captured.append(result_assignments) + async def execute(_tool, _arguments, *, binding=None, result_assignments=None): + captured.append(binding.result_assignments) return {"status": "ok", "updated_variables": []} brain._runtime = BrainRuntime( @@ -2098,7 +2147,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): await brain._enter_action("override_action") await brain._enter_action("none_action") - self.assertEqual(captured, [None, {"choice": "action"}, {}]) + self.assertEqual(captured, [{}, {"choice": "action"}, {}]) async def test_action_client_error_sets_error_status(self): tool = RuntimeTool( @@ -2118,7 +2167,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): { "id": "action", "type": "action", - "data": {"toolId": "client_action"}, + "data": {"toolBinding": {"toolId": "client_action"}}, }, ], "edges": [], @@ -2127,7 +2176,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): ) ) - async def execute(_tool, _arguments, *, result_assignments=None): + async def execute(_tool, _arguments, *, binding=None, result_assignments=None): return { "status": "error", "message": "用户关闭了确认弹窗", @@ -2172,7 +2221,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): { "id": "action", "type": "action", - "data": {"toolId": "client_action"}, + "data": {"toolBinding": {"toolId": "client_action"}}, }, {"id": "end", "type": "end", "data": {}}, ], @@ -2193,7 +2242,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): async def queue_frame(frame): queued.append(frame) - async def execute(_tool, _arguments, *, result_assignments=None): + async def execute(_tool, _arguments, *, binding=None, result_assignments=None): return { "status": "error", "message": "会话已结束", @@ -2244,7 +2293,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "id": "block_action", "type": "action", "data": { - "toolId": "client_action", + "toolBinding": {"toolId": "client_action"}, "userInputPolicy": "block", }, }, @@ -2252,7 +2301,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): "id": "queue_action", "type": "action", "data": { - "toolId": "client_action", + "toolBinding": {"toolId": "client_action"}, "userInputPolicy": "queue", }, }, @@ -2264,7 +2313,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): ) input_states = [] - async def execute(_tool, _arguments, *, result_assignments=None): + async def execute(_tool, _arguments, *, binding=None, result_assignments=None): input_states.append("executing") return {"status": "ok", "updated_variables": []} diff --git a/backend/tests/test_client_tools.py b/backend/tests/test_client_tools.py index 46a5ca4..61bce20 100644 --- a/backend/tests/test_client_tools.py +++ b/backend/tests/test_client_tools.py @@ -1,7 +1,7 @@ import asyncio import unittest -from models import RuntimeTool +from models import RuntimeTool, RuntimeToolBinding from pipecat.frames.frames import ( CancelFrame, EndFrame, @@ -34,9 +34,6 @@ def client_tool() -> RuntimeTool: } ], "timeout_seconds": 3, - "dynamic_variable_assignments": { - "photo_button_visible": "visible" - }, }, }, ) @@ -47,6 +44,9 @@ class FakeClientTools: self.result = result self.calls = [] + def supports(self, _function_name): + return True + async def call( self, function_name, @@ -80,12 +80,9 @@ class ClientToolExecutorTests(unittest.IsolatedAsyncioTestCase): "session", ) - def test_fire_and_forget_rejects_result_assignments(self): - with self.assertRaisesRegex(ValueError, "不能配置结果变量赋值"): - ClientToolConfig( - wait_for_response=False, - dynamic_variable_assignments={"visible": "visible"}, - ) + def test_result_assignments_are_not_a_resource_setting(self): + config = ClientToolConfig(wait_for_response=False) + self.assertNotIn("dynamic_variable_assignments", ClientToolConfig.model_fields) async def test_success_updates_configured_variable(self): store = DynamicVariableStore( @@ -95,7 +92,15 @@ class ClientToolExecutorTests(unittest.IsolatedAsyncioTestCase): port = FakeClientTools({"status": "ok", "data": {"visible": True}}) executor = ToolExecutor(store, client_tools=port) - result = await executor.execute(client_tool(), {"visible": True}) + result = await executor.execute( + client_tool(), + {"visible": True}, + binding=RuntimeToolBinding( + toolId="tool_photo_button", + argumentBindings={"visible": {"source": "model"}}, + resultAssignments={"photo_button_visible": "visible"}, + ), + ) self.assertEqual(result["updated_variables"], ["photo_button_visible"]) self.assertIs(store.values["photo_button_visible"], True) @@ -138,6 +143,18 @@ class ClientToolExecutorTests(unittest.IsolatedAsyncioTestCase): class ClientToolBrokerTests(unittest.IsolatedAsyncioTestCase): + async def test_capability_handshake_rejects_unregistered_tool_immediately(self): + broker = ClientToolBroker() + broker.set_capabilities({"show_message"}) + + with self.assertRaisesRegex(ClientToolError, "未注册"): + await broker.call( + "set_photo_button_visible", + {"visible": True}, + timeout_seconds=30, + ) + self.assertEqual(broker._pending, {}) + async def test_uses_configured_downstream_interrupt_handler(self): broker = ClientToolBroker() outbound = [] diff --git a/backend/tests/test_debug_auto_test.py b/backend/tests/test_debug_auto_test.py index 6fc3b2b..c4d78ed 100644 --- a/backend/tests/test_debug_auto_test.py +++ b/backend/tests/test_debug_auto_test.py @@ -214,7 +214,12 @@ class DebugAutoTestRuntimeTests(unittest.IsolatedAsyncioTestCase): async def test_system_end_tool_keeps_real_lifecycle_in_text_tests(self): cfg = AssistantConfig( type="prompt", - tools=[system_runtime_tool("end_conversation")], + built_in_tools={ + "end_conversation": { + "description": "结束对话", + "capture_reason": True, + } + }, ) brain = build_brain(cfg) llm = FakeLLM() diff --git a/backend/tests/test_prompt_realtime_tools.py b/backend/tests/test_prompt_realtime_tools.py new file mode 100644 index 0000000..b98695f --- /dev/null +++ b/backend/tests/test_prompt_realtime_tools.py @@ -0,0 +1,605 @@ +from __future__ import annotations + +import asyncio +import json +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock + +from fastapi import HTTPException +from models import AssistantConfig, RuntimeTool, RuntimeToolBinding +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + OutputTransportMessageUrgentFrame, +) +from services.brains.base import RealtimeBrainRuntime +from services.brains.prompt_realtime_tools import PromptRealtimeToolController +from services.pipecat.call_lifecycle import ( + CallEndCoordinator, + FixedSpeechPlaybackMarkerFrame, +) +from services.pipecat.realtime_tools import ( + RealtimeToolResult, + RealtimeToolSession, +) +from services.realtime.launcher import validate_runtime_requirements +from services.runtime_variables import DynamicVariableStore +from services.system_tools import RESERVED_TOOL_NAMES +from routes.assistants import _validate_prompt_realtime_tool_capabilities +from routes.model_registry import _ensure_realtime_prompt_tools_remain_supported +from schemas import AssistantUpsert + + +class RealtimeToolSessionTests(unittest.IsolatedAsyncioTestCase): + async def test_follow_up_priority_and_one_response_create(self): + events: list[dict] = [] + suppressed = AsyncMock() + + async def send_event(event: dict) -> None: + events.append(event) + + session = RealtimeToolSession(send_event, suppress_response=suppressed) + + async def dispatch(name, _arguments, _call_id): + return RealtimeToolResult( + {"name": name}, + follow_up={"first": "none", "second": "suppress"}[name], + ) + + session.set_dispatcher(dispatch) + session.submit_call( + response_id="resp", + name="second", + call_id="call_2", + arguments={}, + output_index=1, + ) + session.submit_call( + response_id="resp", + name="first", + call_id="call_1", + arguments={}, + output_index=0, + ) + session.finish_response("resp") + await session.wait_pending() + + self.assertEqual( + [event["item"]["call_id"] for event in events], + ["call_1", "call_2"], + ) + self.assertFalse(any(event["type"] == "response.create" for event in events)) + suppressed.assert_awaited_once() + + async def test_new_input_stops_unstarted_side_effect_and_follow_up(self): + events: list[dict] = [] + first_started = asyncio.Event() + release_first = asyncio.Event() + dispatched: list[str] = [] + + async def send_event(event: dict) -> None: + events.append(event) + + session = RealtimeToolSession(send_event) + + async def dispatch(name, _arguments, _call_id): + dispatched.append(name) + if name == "first": + first_started.set() + await release_first.wait() + return RealtimeToolResult({"status": "ok"}) + + session.set_dispatcher(dispatch) + for index, name in enumerate(("first", "dangerous_second")): + session.submit_call( + response_id="resp", + name=name, + call_id=f"call_{index}", + arguments={}, + output_index=index, + ) + session.finish_response("resp") + await first_started.wait() + session.invalidate_pending_follow_ups() + release_first.set() + await session.wait_pending() + + self.assertEqual(dispatched, ["first"]) + self.assertFalse(any(event["type"] == "response.create" for event in events)) + cancelled = json.loads(events[-1]["item"]["output"]) + self.assertEqual(cancelled["status"], "cancelled") + + async def test_close_cancels_active_background_call(self): + started = asyncio.Event() + cancelled = asyncio.Event() + session = RealtimeToolSession(AsyncMock()) + + async def dispatch(_name, _arguments, _call_id): + started.set() + try: + await asyncio.Future() + except asyncio.CancelledError: + cancelled.set() + raise + + session.set_dispatcher(dispatch) + session.submit_call( + response_id="resp", + name="slow", + call_id="call_slow", + arguments={}, + ) + session.finish_response("resp") + await started.wait() + await session.shutdown() + + self.assertTrue(cancelled.is_set()) + + +class _FakeRealtime: + def __init__(self) -> None: + self.dispatcher = None + self.updated_sessions: list[tuple[str, list]] = [] + self.instructions: list[str] = [] + self.spoken: list[str] = [] + + def set_tool_dispatcher(self, dispatcher) -> None: + self.dispatcher = dispatcher + + async def update_session(self, prompt: str, tools: list) -> None: + self.updated_sessions.append((prompt, tools)) + + async def update_instructions(self, prompt: str) -> None: + self.instructions.append(prompt) + + async def speak_fixed(self, content: str, **_kwargs): + self.spoken.append(content) + future = asyncio.get_running_loop().create_future() + future.set_result(None) + return future + + +class _FakeExecutor: + def __init__(self, result: dict) -> None: + self.result = result + self.calls: list[tuple[str, dict]] = [] + + @staticmethod + def schema_parts(_tool, _binding): + return {"order_id": {"type": "string"}}, ["order_id"] + + async def execute(self, tool, arguments, *, binding): + self.calls.append((tool.function_name, dict(arguments))) + return dict(self.result) + + +class PromptRealtimeToolControllerTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.frames: list = [] + self.ended: list[str] = [] + self.inputs: list[bool] = [] + self.realtime = _FakeRealtime() + + async def queue_frame(self, frame) -> None: + self.frames.append(frame) + if isinstance(frame, FixedSpeechPlaybackMarkerFrame): + await frame.completion.mark_played() + + async def queue_end(self, reason: str) -> None: + self.ended.append(reason) + + def runtime(self, call_end=None, client_tools=None) -> RealtimeBrainRuntime: + return RealtimeBrainRuntime( + realtime=self.realtime, + queue_frame=self.queue_frame, + call_end=call_end or CallEndCoordinator(self.queue_end), + session_id="conv_test", + client_tools=client_tools, + set_input_enabled=self.inputs.append, + ) + + async def controller(self, cfg: AssistantConfig, **runtime_kwargs): + store = DynamicVariableStore.from_config(cfg) + controller = PromptRealtimeToolController( + cfg=cfg, + store=store, + runtime=self.runtime(**runtime_kwargs), + ) + await controller.start() + return controller, store + + async def test_update_state_is_atomic_and_refreshes_prompt_after_event(self): + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + prompt="城市 {{city}} / 确认 {{confirmed}}", + dynamic_variable_definitions={ + "city": {"type": "string"}, + "confirmed": {"type": "boolean"}, + }, + dynamic_variables={"city": "北京", "confirmed": False}, + built_in_tools={ + "updateState": { + "updates": [ + { + "variableName": "city", + "expression": { + "type": "llm", + "parameterName": "new_city", + "valueType": "string", + "description": "新城市", + }, + }, + { + "variableName": "confirmed", + "expression": { + "type": "llm", + "parameterName": "confirmed", + "valueType": "boolean", + "description": "是否确认", + }, + }, + ] + } + }, + ) + controller, store = await self.controller(cfg) + + failed = await controller.dispatch_tool( + "update_state", + {"new_city": "上海", "confirmed": "yes"}, + "call_1", + ) + self.assertEqual(failed.output["status"], "error") + self.assertEqual(store.public_values(), {"city": "北京", "confirmed": False}) + + succeeded = await controller.dispatch_tool( + "update_state", + {"new_city": "上海", "confirmed": True}, + "call_2", + ) + self.assertEqual(succeeded.output["status"], "success") + self.assertEqual(store.public_values(), {"city": "上海", "confirmed": True}) + variable_event = next( + frame.message + for frame in self.frames + if isinstance(frame, OutputTransportMessageUrgentFrame) + and frame.message.get("type") == "session-variables" + ) + self.assertEqual(variable_event["changed"], ["city", "confirmed"]) + self.assertEqual(self.realtime.instructions[-1], "城市 上海 / 确认 true") + + async def test_external_tool_uses_executor_mapping_policy_and_input_gate(self): + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + prompt="状态 {{order_status}}", + dynamic_variable_definitions={"order_status": {"type": "string"}}, + dynamic_variables={"order_status": "unknown"}, + tools=[ + RuntimeTool( + id="tool_order", + name="订单查询", + function_name="lookup_order", + type="http", + definition={ + "type": "http", + "config": { + "allow_interruptions": False, + "execution_mode": "immediate", + }, + }, + ) + ], + tool_bindings=[ + RuntimeToolBinding( + tool_id="tool_order", + argument_bindings={"order_id": {"source": "model"}}, + ) + ], + ) + controller, store = await self.controller(cfg) + store.values["order_status"] = "paid" + controller._executor = _FakeExecutor( + {"status": "ok", "data": {"status": "paid"}, "updated_variables": ["order_status"]} + ) + + result = await controller.dispatch_tool( + "lookup_order", {"order_id": "A-1"}, "call_1" + ) + + self.assertEqual(result.follow_up, "continue") + self.assertEqual(controller._executor.calls, [("lookup_order", {"order_id": "A-1"})]) + self.assertEqual(self.inputs, [False, True]) + self.assertEqual(self.realtime.instructions[-1], "状态 paid") + trace_events = [ + frame.message.get("event") + for frame in self.frames + if isinstance(frame, OutputTransportMessageUrgentFrame) + and frame.message.get("type") == "workflow-event" + ] + self.assertIn("tool_started", trace_events) + self.assertIn("tool_completed", trace_events) + + async def test_missing_client_capability_emits_tool_skipped(self): + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + tools=[ + RuntimeTool( + id="tool_photo", + name="显示拍照按钮", + function_name="set_photo_button_visible", + type="client", + definition={"type": "client", "config": {"parameters": []}}, + ) + ], + tool_bindings=[RuntimeToolBinding(tool_id="tool_photo")], + ) + client_tools = SimpleNamespace(supports=lambda _name: False) + await self.controller(cfg, client_tools=client_tools) + + self.assertEqual(self.realtime.updated_sessions[-1][1], []) + skipped = next( + frame.message + for frame in self.frames + if isinstance(frame, OutputTransportMessageUrgentFrame) + and frame.message.get("event") == "tool_skipped" + ) + self.assertEqual(skipped["reason"], "client_capability_missing") + + async def test_handoff_is_idempotent_and_updates_system_state(self): + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + built_in_tools={"requestHumanHandoff": {}}, + ) + controller, store = await self.controller(cfg) + + await controller.dispatch_tool("request_human_handoff", {"reason": "投诉"}, "1") + await controller.dispatch_tool("request_human_handoff", {"reason": "再次"}, "2") + + handoff_events = [ + frame.message + for frame in self.frames + if isinstance(frame, OutputTransportMessageUrgentFrame) + and frame.message.get("type") == "handoff-requested" + ] + self.assertEqual(len(handoff_events), 1) + self.assertEqual(store.values["system__handoff_status"], "requested") + + async def test_custom_end_waits_for_playback_marker_before_hangup(self): + call_end = CallEndCoordinator(self.queue_end) + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + built_in_tools={ + "endConversation": { + "messageType": "custom", + "customMessage": "感谢来电，再见。", + "captureReason": True, + "allowEndInterruption": False, + } + }, + ) + controller, _store = await self.controller(cfg, call_end=call_end) + + result = await controller.dispatch_tool( + "end_conversation", {"reason": "completed"}, "call_end" + ) + self.assertEqual(result.follow_up, "suppress") + self.assertEqual(self.ended, []) + await result.after_output() + + self.assertEqual(self.realtime.spoken, ["感谢来电，再见。"]) + self.assertEqual(self.ended, ["completed"]) + self.assertIn(False, self.inputs) + + async def test_generated_end_is_interruptible_and_idempotent(self): + call_end = CallEndCoordinator(self.queue_end) + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + built_in_tools={ + "endConversation": { + "messageType": "none", + "allowEndInterruption": True, + } + }, + ) + controller, _store = await self.controller(cfg, call_end=call_end) + + first = await controller.dispatch_tool( + "end_conversation", {"reason": "done"}, "call_1" + ) + second = await controller.dispatch_tool( + "end_conversation", {"reason": "done"}, "call_2" + ) + self.assertEqual(first.follow_up, "continue") + self.assertEqual(second.follow_up, "suppress") + await controller.on_user_speech_started() + + self.assertFalse(call_end.ending) + self.assertEqual(self.ended, []) + + async def test_generated_end_waits_for_its_own_transport_speech(self): + call_end = CallEndCoordinator(self.queue_end) + # Simulate completed speech from an older response. The new response + # must reset this state before deciding whether it can hang up. + await call_end.observe(BotStartedSpeakingFrame()) + await call_end.observe(BotStoppedSpeakingFrame()) + cfg = AssistantConfig( + type="prompt", + runtimeMode="realtime", + realtime_interface_type="qwen-audio-realtime", + built_in_tools={"endConversation": {"messageType": "none"}}, + ) + controller, _store = await self.controller(cfg, call_end=call_end) + await controller.dispatch_tool( + "end_conversation", {"reason": "done"}, "call_1" + ) + + controller.on_assistant_text_start() + await controller.on_response_done("再见。", False) + self.assertEqual(self.ended, []) + + await call_end.observe(BotStartedSpeakingFrame()) + self.assertEqual(self.ended, []) + await call_end.observe(BotStoppedSpeakingFrame()) + self.assertEqual(self.ended, ["done"]) + + +class PromptRealtimeCapabilityTests(unittest.TestCase): + @staticmethod + def config(interface_type: str, **updates) -> AssistantConfig: + values = { + "type": "prompt", + "runtimeMode": "realtime", + "realtime_interface_type": interface_type, + "realtimeModel": "model", + "realtime_base_url": "wss://example.test/realtime", + "realtime_api_key": "secret", + } + values.update(updates) + return AssistantConfig(**values) + + def test_qwen_accepts_supported_tools_and_rejects_skip_turn(self): + validate_runtime_requirements( + self.config( + "qwen-audio-realtime", + built_in_tools={"updateState": {"updates": []}}, + ) + ) + with self.assertRaisesRegex(ValueError, "skipTurn"): + validate_runtime_requirements( + self.config( + "qwen-audio-realtime", + built_in_tools={"skipTurn": {}}, + ) + ) + + def test_stepfun_rejects_prompt_realtime_tools(self): + with self.assertRaisesRegex(ValueError, "endConversation"): + validate_runtime_requirements( + self.config( + "stepfun-realtime", + built_in_tools={"endConversation": {}}, + ) + ) + + def test_runtime_rejects_reserved_external_names(self): + self.assertIn("end_conversation", RESERVED_TOOL_NAMES) + with self.assertRaisesRegex(ValueError, "平台保留名"): + validate_runtime_requirements( + self.config( + "qwen-audio-realtime", + tools=[ + RuntimeTool( + id="legacy", + name="旧外部工具", + function_name="end_conversation", + type="http", + ) + ], + tool_bindings=[RuntimeToolBinding(tool_id="legacy")], + ) + ) + + +class _ScalarRows: + def __init__(self, rows) -> None: + self._rows = rows + + def scalars(self): + return self + + def all(self): + return list(self._rows) + + +class PromptRealtimeApiGateTests(unittest.IsolatedAsyncioTestCase): + async def test_save_accepts_qwen_features_and_rejects_stepfun_or_skip_turn(self): + resource = SimpleNamespace( + enabled=True, + capability="Realtime", + interface_type="qwen-audio-realtime", + ) + qwen = SimpleNamespace( + enabled=True, + field_schema={ + "features": { + "realtimePromptTools": { + "externalTypes": ["http", "mcp", "client"], + "builtIns": [ + "endConversation", + "updateState", + "requestHumanHandoff", + ], + } + } + }, + ) + session = AsyncMock() + + async def get(_model, key): + return resource if key == "model_qwen" else qwen + + session.get.side_effect = get + supported = AssistantUpsert( + name="Qwen", + type="prompt", + runtimeMode="realtime", + modelResourceIds={"Realtime": "model_qwen"}, + builtInTools={"updateState": {"updates": []}}, + ) + await _validate_prompt_realtime_tool_capabilities(session, supported) + + skipped = AssistantUpsert( + name="Qwen", + type="prompt", + runtimeMode="realtime", + modelResourceIds={"Realtime": "model_qwen"}, + builtInTools={"skipTurn": {"description": "silent"}}, + ) + with self.assertRaisesRegex(HTTPException, "skipTurn"): + await _validate_prompt_realtime_tool_capabilities(session, skipped) + + resource.interface_type = "stepfun-realtime" + stepfun = SimpleNamespace(enabled=True, field_schema={"fields": []}) + session.get.side_effect = lambda _model, key: ( + resource if key == "model_qwen" else stepfun + ) + with self.assertRaisesRegex(HTTPException, "updateState"): + await _validate_prompt_realtime_tool_capabilities(session, supported) + + async def test_model_resource_cannot_drop_features_while_tools_are_bound(self): + assistant = SimpleNamespace( + id="asst_qwen", + name="Realtime 客服", + built_in_tools={"end_conversation": {"capture_reason": True}}, + ) + session = AsyncMock() + session.execute.return_value = _ScalarRows([assistant]) + resource = SimpleNamespace(id="model_qwen", capability="Realtime") + unsupported_definition = SimpleNamespace( + capability="Realtime", + field_schema={"fields": []}, + ) + + with self.assertRaisesRegex(HTTPException, "新接口不支持其内建工具"): + await _ensure_realtime_prompt_tools_remain_supported( + session, + resource, + unsupported_definition, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_qwen_audio_realtime.py b/backend/tests/test_qwen_audio_realtime.py index 4bbce8e..7556a49 100644 --- a/backend/tests/test_qwen_audio_realtime.py +++ b/backend/tests/test_qwen_audio_realtime.py @@ -10,6 +10,7 @@ from pipecat.frames.frames import OutputTransportMessageUrgentFrame, TTSAudioRaw from services import model_resource_tester from services.interface_catalog import INTERFACE_DEFINITIONS from services.pipecat.qwen_audio_realtime import QwenAudioRealtimeService +from services.pipecat.realtime_tools import RealtimeTool, RealtimeToolResult from services.pipecat.service_factory import ( create_realtime_service, realtime_audio_sample_rates, @@ -63,9 +64,160 @@ class QwenAudioRealtimeServiceTest(unittest.IsolatedAsyncioTestCase): "silence_duration_ms": 800, }, "max_history_turns": 20, + "tools": [], + "tool_choice": "auto", }, ) + async def test_session_update_uses_qwen_nested_function_schema(self): + service = _service() + websocket = _OpenWebSocket() + service._websocket = websocket + service._session_ready.set() + tool = RealtimeTool( + name="lookup_order", + description="查询订单", + properties={"order_id": {"type": "string"}}, + required=("order_id",), + ) + + await service.update_session("Use tools.", [tool]) + + self.assertEqual( + websocket.messages[-1]["session"]["tools"], + [ + { + "type": "function", + "function": { + "name": "lookup_order", + "description": "查询订单", + "parameters": { + "type": "object", + "properties": { + "order_id": {"type": "string"} + }, + "required": ["order_id"], + }, + }, + } + ], + ) + + async def test_multiple_function_calls_are_flushed_once_in_output_order(self): + service = _service() + websocket = _OpenWebSocket() + service._websocket = websocket + service._session_ready.set() + service.push_frame = AsyncMock() + calls: list[tuple[str, dict]] = [] + + async def dispatch(name, arguments, _call_id): + calls.append((name, arguments)) + return RealtimeToolResult({"status": "ok", "name": name}) + + service.set_tool_dispatcher(dispatch) + await service._handle_server_event( + {"type": "response.created", "response": {"id": "resp_1"}} + ) + await service._handle_server_event( + { + "type": "response.function_call_arguments.done", + "response_id": "resp_1", + "output_index": 1, + "call_id": "call_2", + "name": "second", + "arguments": '{"position":2}', + } + ) + await service._handle_server_event( + { + "type": "response.function_call_arguments.done", + "response_id": "resp_1", + "output_index": 0, + "call_id": "call_1", + "name": "first", + "arguments": '{"position":1}', + } + ) + # Qwen can emit output_item.done for the same call as arguments.done. + await service._handle_server_event( + { + "type": "response.output_item.done", + "response_id": "resp_1", + "output_index": 0, + "item": { + "type": "function_call", + "call_id": "call_1", + "name": "first", + "arguments": '{"position":1}', + }, + } + ) + await service._handle_server_event( + { + "type": "response.done", + "response": {"id": "resp_1", "status": "completed"}, + } + ) + await service._tool_session.wait_pending() + + self.assertEqual( + calls, + [("first", {"position": 1}), ("second", {"position": 2})], + ) + outputs = [ + item + for item in websocket.messages + if item["type"] == "conversation.item.create" + and item["item"]["type"] == "function_call_output" + ] + self.assertEqual( + [item["item"]["call_id"] for item in outputs], + ["call_1", "call_2"], + ) + self.assertEqual( + sum(item["type"] == "response.create" for item in websocket.messages), + 1, + ) + + async def test_malformed_arguments_do_not_execute_side_effect(self): + service = _service() + websocket = _OpenWebSocket() + service._websocket = websocket + service._session_ready.set() + service.push_frame = AsyncMock() + dispatcher = AsyncMock() + service.set_tool_dispatcher(dispatcher) + + await service._handle_server_event( + {"type": "response.created", "response": {"id": "resp_bad"}} + ) + await service._handle_server_event( + { + "type": "response.function_call_arguments.done", + "response_id": "resp_bad", + "call_id": "call_bad", + "name": "end_conversation", + "arguments": "{bad json", + } + ) + await service._handle_server_event( + { + "type": "response.done", + "response": {"id": "resp_bad", "status": "completed"}, + } + ) + await service._tool_session.wait_pending() + + dispatcher.assert_not_awaited() + output = next( + item["item"]["output"] + for item in websocket.messages + if item["type"] == "conversation.item.create" + and item["item"]["type"] == "function_call_output" + ) + self.assertIn("JSON", json.loads(output)["message"]) + def test_smart_turn_and_value_ranges_are_explicit(self): service = _service(turn_detection_mode="smart_turn") self.assertEqual(service._turn_detection_config(), {"type": "smart_turn"}) diff --git a/backend/tests/test_startup_actions.py b/backend/tests/test_startup_actions.py index 728c71a..da6d0f3 100644 --- a/backend/tests/test_startup_actions.py +++ b/backend/tests/test_startup_actions.py @@ -5,6 +5,7 @@ from types import SimpleNamespace from fastapi import HTTPException from routes.assistants import _validate_startup_actions +from models import RuntimeToolBinding from schemas import AssistantUpsert from services.config_resolver import _runtime_tool_ids @@ -18,25 +19,28 @@ def startup_body( name="启动动作测试", type="prompt", runtimeMode="pipeline", - toolIds=["tool_message"] if bind_tool else [], + toolBindings=( + [{"toolId": "tool_message", "argumentBindings": {}}] + if bind_tool + else [] + ), startup={ "executionMode": "sequential", "actions": [ { "id": "opening_message", "phase": phase, - "toolId": "tool_message", - "arguments": { - "title": "重要提示", - "message": "请确认已阅读。", - "actions": [ - { - "id": "confirmed", - "label": "确认", - "style": "primary", - } - ], - "dismissible": False, + "toolBinding": { + "toolId": "tool_message", + "argumentBindings": { + "title": {"source": "literal", "value": "重要提示"}, + "message": {"source": "literal", "value": "请确认已阅读。"}, + "actions": { + "source": "literal", + "value": [{"id": "confirmed", "label": "确认", "style": "primary"}], + }, + "dismissible": {"source": "literal", "value": False}, + }, }, "required": True, } @@ -57,11 +61,14 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase): def test_runtime_pool_includes_bound_and_startup_only_tools(self): self.assertEqual( _runtime_tool_ids( - ["conversation_tool", "shared_tool"], + [ + RuntimeToolBinding(tool_id="conversation_tool"), + RuntimeToolBinding(tool_id="shared_tool"), + ], { "actions": [ - {"tool_id": "opening_tool"}, - {"toolId": "shared_tool"}, + {"toolBinding": {"toolId": "opening_tool"}}, + {"toolBinding": {"toolId": "shared_tool"}}, ] }, ), @@ -129,7 +136,7 @@ class StartupActionValidationTests(unittest.IsolatedAsyncioTestCase): body = startup_body(bind_tool=False) await _validate_startup_actions(FakeSession(tool), body) - self.assertEqual(body.tool_ids, []) + self.assertEqual(body.tool_bindings, []) async def test_show_message_requires_session_wait(self): tool = SimpleNamespace( diff --git a/backend/tests/test_tool_executor_mcp.py b/backend/tests/test_tool_executor_mcp.py index 5f606d3..8a541aa 100644 --- a/backend/tests/test_tool_executor_mcp.py +++ b/backend/tests/test_tool_executor_mcp.py @@ -1,6 +1,6 @@ import unittest -from models import RuntimeMcpServer, RuntimeTool +from models import RuntimeMcpServer, RuntimeTool, RuntimeToolBinding from services.runtime_variables import DynamicVariableStore from services.tool_executor import ToolExecutionError, ToolExecutor from services.tools import McpClientError @@ -28,16 +28,13 @@ def mcp_tool() -> RuntimeTool: }, "required": ["order_id"], }, - "dynamic_variable_assignments": { - "order_status": "structuredContent.status" - }, }, }, mcp_server=RuntimeMcpServer( id="mcp_orders", name="订单服务", - url="https://mcp.example.com/{{tenant}}", - headers={"X-Tenant": "{{tenant}}"}, + url="https://mcp.example.com/{{input__tenant}}", + headers={"X-Tenant": "{{input__tenant}}"}, secret_headers={"Authorization": "Bearer secret"}, ), ) @@ -77,6 +74,14 @@ class ToolExecutorMcpTests(unittest.IsolatedAsyncioTestCase): result = await executor.execute( mcp_tool(), {"order_id": "A-100"}, + binding=RuntimeToolBinding( + toolId="tool_mcp_order", + argumentBindings={"order_id": {"source": "model"}}, + templateBindings={ + "tenant": {"source": "variable", "variableName": "tenant"} + }, + resultAssignments={"order_status": "structuredContent.status"}, + ), ) self.assertEqual(result["status"], "ok") @@ -98,7 +103,20 @@ class ToolExecutorMcpTests(unittest.IsolatedAsyncioTestCase): ) with self.assertRaisesRegex(ToolExecutionError, "远端不可用"): - await executor.execute(mcp_tool(), {"order_id": "A-100"}) + await executor.execute( + mcp_tool(), + {"order_id": "A-100"}, + binding=RuntimeToolBinding( + toolId="tool_mcp_order", + argumentBindings={"order_id": {"source": "model"}}, + templateBindings={ + "tenant": { + "source": "variable", + "variableName": "tenant", + } + }, + ), + ) if __name__ == "__main__": diff --git a/backend/tests/test_tool_organization_v4.py b/backend/tests/test_tool_organization_v4.py new file mode 100644 index 0000000..d98f507 --- /dev/null +++ b/backend/tests/test_tool_organization_v4.py @@ -0,0 +1,379 @@ +from __future__ import annotations + +import importlib +import unittest + +from fastapi import HTTPException +from models import RuntimeTool, RuntimeToolBinding +from routes.tools import _payload +from schemas import AssistantOut, AssistantUpsert, ToolUpsert +from services.runtime_variables import DynamicVariableStore +from services.state_expressions import ( + StateExpressionError, + evaluate_update_rules, + validate_update_rules, +) +from services.system_tools import runtime_system_tools +from services.tool_executor import ToolExecutionError, ToolExecutor + + +class ToolOrganizationContractTests(unittest.TestCase): + def test_system_is_not_a_resource_type(self): + with self.assertRaises(ValueError): + ToolUpsert( + name="结束对话", + functionName="end_conversation", + definition={ + "schemaVersion": 2, + "type": "system", + "config": {}, + }, + ) + + def test_prompt_keeps_resources_and_builtins_separate(self): + assistant = AssistantUpsert( + name="客服", + type="prompt", + toolBindings=[ + { + "toolId": "tool_lookup", + "argumentBindings": {"order_id": {"source": "model"}}, + } + ], + builtInTools={"endConversation": {"captureReason": True}}, + ) + + self.assertEqual(assistant.tool_bindings[0].tool_id, "tool_lookup") + tools = runtime_system_tools(assistant.built_in_tools.model_dump()) + self.assertEqual([tool.function_name for tool in tools], ["end_conversation"]) + + def test_realtime_prompt_request_and_response_preserve_builtins(self): + data = { + "name": "历史 Realtime 助手", + "type": "prompt", + "runtimeMode": "realtime", + "builtInTools": {"endConversation": {"captureReason": True}}, + } + + request = AssistantUpsert(**data) + self.assertIsNotNone(request.built_in_tools.end_conversation) + + response = AssistantOut(id="asst_legacy", **data) + self.assertIsNotNone(response.built_in_tools.end_conversation) + + def test_workflow_camel_case_builtin_config_is_normalized_for_runtime(self): + [tool] = runtime_system_tools( + { + "endConversation": { + "messageType": "custom", + "customMessage": "再见", + "captureReason": False, + "allowEndInterruption": True, + } + } + ) + + config = tool.definition["config"] + self.assertEqual(config["message_type"], "custom") + self.assertEqual(config["custom_message"], "再见") + self.assertFalse(config["capture_reason"]) + self.assertTrue(config["allow_end_interruption"]) + + def test_workflow_public_boundary_is_v4(self): + assistant = AssistantUpsert( + name="工作流", + type="workflow", + graph={"specVersion": 4, "settings": {}, "nodes": [], "edges": []}, + ) + self.assertEqual(assistant.graph["specVersion"], 4) + + def test_http_resource_derives_explicit_template_inputs(self): + body = ToolUpsert( + name="订单查询", + functionName="query_order", + definition={ + "schemaVersion": 2, + "type": "http", + "config": { + "url": "https://example.com/{{input__tenant}}/orders", + "headers": {}, + "body": {}, + }, + }, + ) + + payload = _payload(body) + + self.assertEqual( + payload["definition"]["config"]["template_inputs"][0]["name"], + "tenant", + ) + + def test_http_resource_rejects_assistant_variable_placeholder(self): + body = ToolUpsert( + name="订单查询", + functionName="query_order", + definition={ + "schemaVersion": 2, + "type": "http", + "config": { + "url": "https://example.com/{{tenant}}/orders", + "headers": {}, + "body": {}, + }, + }, + ) + + with self.assertRaisesRegex(HTTPException, "input__name"): + _payload(body) + + def test_http_resource_rejects_non_local_secret_name(self): + body = ToolUpsert( + name="订单查询", + functionName="query_order", + definition={ + "schemaVersion": 2, + "type": "http", + "config": { + "url": "https://example.com/orders", + "headers": {}, + "body": {}, + }, + }, + secrets={"dynamic_variables": {"api_token": "secret"}}, + ) + + with self.assertRaisesRegex(HTTPException, "secret__"): + _payload(body) + + def test_http_resource_keeps_secret_placeholders_in_headers(self): + body = ToolUpsert( + name="订单查询", + functionName="query_order", + definition={ + "schemaVersion": 2, + "type": "http", + "config": { + "url": "https://example.com/orders", + "headers": {"Authorization": "Bearer {{secret__token}}"}, + "body": {}, + }, + }, + secrets={"dynamic_variables": {"secret__token": "secret"}}, + ) + + payload = _payload(body) + + self.assertEqual( + payload["definition"]["config"]["headers"]["Authorization"], + "Bearer {{secret__token}}", + ) + + def test_migration_preserves_inherited_update_state_scope(self): + migration = importlib.import_module( + "migrations.versions.20260813_0021_organize_tool_resources" + ) + settings = { + "runtimeMode": "pipeline", + "globalPrompt": "全局规则", + "defaultLlmResourceId": "llm_default", + "toolBindings": [{"toolId": "tool_lookup"}], + "builtInTools": { + "updateState": { + "updates": [ + { + "variableName": "name", + "expression": {"type": "constant", "value": "all"}, + }, + { + "variableName": "city", + "expression": {"type": "constant", "value": "all"}, + }, + ] + } + }, + } + data = {"prompt": "节点任务", "inheritGlobalConfig": True} + + migration._freeze_inherited_agent_for_update_state( + data, + settings, + {"name": {"type": "string"}, "city": {"type": "string"}}, + ["city"], + ) + + self.assertFalse(data["inheritGlobalConfig"]) + self.assertEqual(data["llmResourceId"], "llm_default") + self.assertEqual(data["prompt"], "全局规则\n\n节点任务") + self.assertEqual( + [ + rule["variableName"] + for rule in data["builtInTools"]["updateState"]["updates"] + ], + ["city"], + ) + self.assertEqual(len(settings["builtInTools"]["updateState"]["updates"]), 2) + + def test_realtime_migration_does_not_create_model_overrides(self): + migration = importlib.import_module( + "migrations.versions.20260813_0021_organize_tool_resources" + ) + settings = { + "runtimeMode": "realtime", + "defaultLlmResourceId": "unused", + "enableInterrupt": True, + "toolBindings": [], + "builtInTools": {"updateState": {"updates": []}}, + } + data = {"inheritGlobalConfig": True} + + migration._freeze_inherited_agent_for_update_state( + data, + settings, + {"name": {"type": "string"}}, + ["name"], + ) + + self.assertNotIn("llmResourceId", data) + self.assertNotIn("enableInterrupt", data) + + +class ToolBindingTests(unittest.IsolatedAsyncioTestCase): + async def test_only_model_sources_are_visible_to_llm(self): + tool = RuntimeTool( + id="tool_order", + name="查询订单", + function_name="query_order", + type="http", + definition={ + "config": { + "parameters": [ + {"name": "order_id", "type": "string", "required": True}, + {"name": "tenant", "type": "string", "required": True}, + ] + } + }, + ) + binding = RuntimeToolBinding( + toolId=tool.id, + argumentBindings={ + "order_id": {"source": "model"}, + "tenant": {"source": "variable", "variableName": "tenant"}, + }, + ) + + properties, required = ToolExecutor.schema_parts(tool, binding) + + self.assertEqual(set(properties), {"order_id"}) + self.assertEqual(required, ["order_id"]) + executor = ToolExecutor(DynamicVariableStore({"tenant": "school-a"})) + self.assertEqual( + executor.resolve_arguments(tool, binding, {"order_id": "A-1"}), + {"order_id": "A-1", "tenant": "school-a"}, + ) + + async def test_model_cannot_override_non_model_binding(self): + tool = RuntimeTool( + id="tool_order", + name="查询订单", + function_name="query_order", + type="http", + definition={ + "config": { + "parameters": [ + {"name": "tenant", "type": "string", "required": True} + ] + } + }, + ) + binding = RuntimeToolBinding( + toolId=tool.id, + argumentBindings={ + "tenant": {"source": "variable", "variableName": "tenant"} + }, + ) + executor = ToolExecutor(DynamicVariableStore({"tenant": "school-a"})) + + with self.assertRaisesRegex(ToolExecutionError, "未授权"): + executor.resolve_arguments(tool, binding, {"tenant": "attacker"}) + + +class UpdateStateExpressionTests(unittest.TestCase): + def test_expression_tree_reads_one_snapshot(self): + definitions = { + "visits": {"type": "number"}, + "vip": {"type": "boolean"}, + } + rules = [ + { + "variableName": "visits", + "expression": { + "type": "add", + "operands": [ + {"type": "variable", "variableName": "visits"}, + {"type": "constant", "value": 1}, + ], + }, + }, + { + "variableName": "vip", + "expression": { + "type": "gte", + "operands": [ + {"type": "variable", "variableName": "visits"}, + {"type": "constant", "value": 3}, + ], + }, + }, + ] + + self.assertEqual( + validate_update_rules(rules, definitions, allow_llm=False), + {}, + ) + self.assertEqual( + evaluate_update_rules(rules, {"visits": 3, "vip": False}), + {"visits": 4, "vip": True}, + ) + + def test_deterministic_node_rejects_llm_leaf(self): + with self.assertRaisesRegex(StateExpressionError, "不能使用 LLM"): + validate_update_rules( + [ + { + "variableName": "name", + "expression": { + "type": "llm", + "parameterName": "name", + "valueType": "string", + "description": "姓名", + }, + } + ], + {"name": {"type": "string"}}, + allow_llm=False, + ) + + def test_llm_parameter_cannot_drive_multiple_update_targets(self): + expression = { + "type": "llm", + "parameterName": "shared", + "valueType": "string", + "description": "提取文本", + } + with self.assertRaisesRegex(StateExpressionError, "多个变量"): + validate_update_rules( + [ + {"variableName": "first", "expression": expression}, + {"variableName": "second", "expression": expression}, + ], + { + "first": {"type": "string"}, + "second": {"type": "string"}, + }, + allow_llm=True, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/backend/tests/test_workflow_v3.py b/backend/tests/test_workflow_v3.py index 100b69c..71ff72d 100644 --- a/backend/tests/test_workflow_v3.py +++ b/backend/tests/test_workflow_v3.py @@ -9,7 +9,7 @@ from unittest.mock import AsyncMock from models import AssistantConfig, RuntimeModelResource from fastapi import HTTPException from routes.assistants import ( - _validate_system_tool_selection, + _validate_update_state_config, _validate_workflow, _validate_workflow_references, ) @@ -23,7 +23,7 @@ from services.workflow_engine import WorkflowEngine def valid_graph(): return { - "specVersion": 3, + "specVersion": 4, "settings": {"globalPrompt": "服务 {{customer}}"}, "nodes": [ {"id": "start", "type": "start", "data": {"name": "Start"}}, @@ -149,20 +149,20 @@ class WorkflowGraphTests(unittest.TestCase): self.assertNotIn("transitionSpeech", edge["data"]) self.assertNotIn("transition_speech", edge["data"]) - def test_action_defaults_preserve_legacy_result_assignment_behavior(self): + def test_action_defaults_use_one_binding_contract(self): graph = valid_graph() graph["nodes"].extend( [ { "id": "legacy_override", "type": "action", - "data": {"resultAssignments": {"status": "order.status"}}, + "data": {"toolBinding": {"toolId": "lookup"}}, }, {"id": "legacy_none", "type": "action", "data": {}}, { "id": "explicit_inherit", "type": "action", - "data": {"resultAssignmentMode": "inherit"}, + "data": {"toolBinding": {"toolId": "notify"}}, }, ] ) @@ -174,14 +174,9 @@ class WorkflowGraphTests(unittest.TestCase): if node["type"] == "action" } - self.assertEqual( - actions["legacy_override"]["resultAssignmentMode"], "override" - ) - self.assertEqual(actions["legacy_none"]["resultAssignmentMode"], "none") - self.assertEqual( - actions["explicit_inherit"]["resultAssignmentMode"], "inherit" - ) - self.assertEqual(actions["legacy_none"]["resultAssignments"], {}) + self.assertEqual(actions["legacy_override"]["toolBinding"]["toolId"], "lookup") + self.assertEqual(actions["legacy_none"]["toolBinding"]["toolId"], "") + self.assertEqual(actions["explicit_inherit"]["toolBinding"]["toolId"], "notify") self.assertEqual(actions["legacy_none"]["userInputPolicy"], "queue") self.assertNotIn("speech", actions["legacy_none"]) @@ -192,16 +187,14 @@ class WorkflowGraphTests(unittest.TestCase): "id": "action", "type": "action", "data": { - "resultAssignmentMode": "unexpected", - "resultAssignments": [], + "toolBinding": {}, "userInputPolicy": "cancel", }, } ) errors = validate_graph(graph) - self.assertTrue(any("结果变量模式无效" in error for error in errors)) - self.assertTrue(any("结果变量映射必须是对象" in error for error in errors)) + self.assertTrue(any("必须配置工具绑定" in error for error in errors)) self.assertTrue(any("用户输入策略无效" in error for error in errors)) def test_message_defaults_and_validation(self): @@ -299,7 +292,7 @@ class WorkflowGraphTests(unittest.TestCase): "defaultTtsResourceId": "tts_global", "visionEnabled": True, "visionModelResourceId": "vision_global", - "toolIds": ["tool_global"], + "toolBindings": [{"toolId": "tool_global"}], "knowledgeBaseId": "kb_global", } ) @@ -312,7 +305,7 @@ class WorkflowGraphTests(unittest.TestCase): "ttsResourceId": "tts_agent", "visionEnabled": True, "visionModelResourceId": "vision_agent", - "toolIds": ["tool_agent"], + "toolBindings": [{"toolId": "tool_agent"}], "knowledgeBaseId": "kb_agent", } ) @@ -337,7 +330,7 @@ class WorkflowGraphTests(unittest.TestCase): def test_existing_agent_override_disables_implicit_inheritance(self): graph = valid_graph() agent = next(node for node in graph["nodes"] if node["type"] == "agent") - agent["data"]["toolIds"] = ["legacy_tool"] + agent["data"]["toolBindings"] = [{"toolId": "legacy_tool"}] normalized = normalize_graph(graph) normalized_agent = next( node for node in normalized["nodes"] if node["type"] == "agent" @@ -353,7 +346,7 @@ class WorkflowGraphTests(unittest.TestCase): "llmResourceId": "stale_llm", "asrResourceId": "stale_asr", "ttsResourceId": "stale_tts", - "toolIds": ["stale_tool"], + "toolBindings": [{"toolId": "stale_tool"}], "knowledgeBaseId": "stale_kb", } ) @@ -367,11 +360,6 @@ class WorkflowGraphTests(unittest.TestCase): def test_agent_effective_config_inherits_then_switches_to_override(self): graph = valid_graph() agent = next(node for node in graph["nodes"] if node["id"] == "agent") - agent["data"].update( - { - "stateVariableNames": ["customer", "order_status"], - } - ) graph["settings"].update( { "defaultLlmResourceId": "llm_global", @@ -379,7 +367,11 @@ class WorkflowGraphTests(unittest.TestCase): "defaultTtsResourceId": "tts_global", "visionEnabled": True, "visionModelResourceId": "vision_global", - "toolIds": ["tool_global", "update_state", "skip_turn"], + "toolBindings": [{"toolId": "tool_global"}], + "builtInTools": { + "updateState": {"description": "更新", "updates": []}, + "skipTurn": {"description": "跳过"}, + }, "knowledgeBaseId": "kb_global", "knowledgeMode": "on_demand", "knowledgeTopN": 8, @@ -401,13 +393,10 @@ class WorkflowGraphTests(unittest.TestCase): ) self.assertTrue(engine.uses_vision()) self.assertEqual( - inherited.tool_ids, - ("tool_global", "update_state", "skip_turn"), - ) - self.assertEqual( - inherited.state_variable_names, - ("customer", "order_status"), + [binding["toolId"] for binding in inherited.tool_bindings], + ["tool_global"], ) + self.assertIn("updateState", inherited.built_in_tools) self.assertEqual(inherited.knowledge_mode, "on_demand") self.assertFalse(inherited.enable_interrupt) self.assertEqual( @@ -419,7 +408,8 @@ class WorkflowGraphTests(unittest.TestCase): { "inheritGlobalConfig": False, "llmResourceId": "llm_agent", - "toolIds": ["tool_agent", "update_state", "skip_turn"], + "toolBindings": [{"toolId": "tool_agent"}], + "builtInTools": {"skipTurn": {"description": "跳过"}}, "knowledgeBaseId": "", "visionEnabled": False, "visionModelResourceId": "", @@ -436,8 +426,8 @@ class WorkflowGraphTests(unittest.TestCase): self.assertIsNone(custom.vision_model_resource_id) self.assertFalse(engine.uses_vision()) self.assertEqual( - custom.tool_ids, - ("tool_agent", "update_state", "skip_turn"), + [binding["toolId"] for binding in custom.tool_bindings], + ["tool_agent"], ) self.assertEqual(custom.knowledge_mode, "disabled") self.assertTrue(custom.enable_interrupt) @@ -455,7 +445,12 @@ class WorkflowGraphTests(unittest.TestCase): "type": "update_state", "data": { "name": "记录状态", - "assignments": {"confirmed": True}, + "updates": [ + { + "variableName": "confirmed", + "expression": {"type": "constant", "value": True}, + } + ], }, }, ) @@ -469,12 +464,6 @@ class WorkflowGraphTests(unittest.TestCase): "data": {"mode": "always", "priority": 0}, }, ) - agent = next(node for node in graph["nodes"] if node["id"] == "agent") - agent["data"].update( - { - "stateVariableNames": ["customer"], - } - ) body = AssistantUpsert( name="状态工作流", type="workflow", @@ -494,45 +483,23 @@ class WorkflowGraphTests(unittest.TestCase): ) _validate_workflow(body) + _validate_update_state_config(body) normalized_state = next( node for node in body.graph["nodes"] if node["type"] == "update_state" ) - self.assertEqual(normalized_state["data"]["assignments"], {"confirmed": True}) - - normalized_state["data"]["assignments"] = {"missing": "x"} - with self.assertRaisesRegex(HTTPException, "未声明变量:missing"): - _validate_workflow(body) - - def test_agent_update_state_tool_requires_an_authorized_variable(self): - graph = valid_graph() - graph["settings"]["toolIds"] = ["tool_update_state"] - agent = next(node for node in graph["nodes"] if node["id"] == "agent") - body = AssistantUpsert( - name="缺少授权", - type="workflow", - dynamicVariableDefinitions={ - "customer": { - "type": "string", - "required": False, - "default": None, - } - }, - graph=graph, + self.assertEqual( + normalized_state["data"]["updates"][0]["variableName"], + "confirmed", ) - _validate_workflow(body) - system_tool = SimpleNamespace( - id="tool_update_state", - status="active", - type="system", - definition={"config": {"kind": "update_state"}}, - ) - result = SimpleNamespace( - scalars=lambda: SimpleNamespace(all=lambda: [system_tool]) - ) - session = SimpleNamespace(execute=AsyncMock(return_value=result)) - with self.assertRaisesRegex(HTTPException, "必须授权至少一个变量"): - asyncio.run(_validate_system_tool_selection(session, body)) + normalized_state["data"]["updates"] = [ + { + "variableName": "missing", + "expression": {"type": "constant", "value": "x"}, + } + ] + with self.assertRaisesRegex(HTTPException, "目标未声明"): + _validate_update_state_config(body) def test_vision_resource_creates_isolated_runtime_config(self): base = AssistantConfig(type="workflow", model="text-only") @@ -610,7 +577,7 @@ class WorkflowGraphTests(unittest.TestCase): "settings": {}, "nodes": [ {"id": "start", "type": "start", "data": {}}, - {"id": "action", "type": "action", "data": {}}, + {"id": "action", "type": "action", "data": {"toolBinding": {"toolId": "tool"}}}, ], "edges": [ { @@ -672,7 +639,7 @@ class WorkflowGraphTests(unittest.TestCase): "nodes": [ {"id": "start", "type": "start", "data": {}}, {"id": "agent", "type": "agent", "data": {}}, - {"id": "action", "type": "action", "data": {}}, + {"id": "action", "type": "action", "data": {"toolBinding": {"toolId": "tool"}}}, {"id": "end", "type": "end", "data": {}}, ], "edges": [ @@ -686,7 +653,7 @@ class WorkflowGraphTests(unittest.TestCase): "nodes": [ {"id": "start", "type": "start", "data": {}}, {"id": "agent", "type": "agent", "data": {}}, - {"id": "action", "type": "action", "data": {}}, + {"id": "action", "type": "action", "data": {"toolBinding": {"toolId": "tool"}}}, {"id": "handoff", "type": "handoff", "data": {}}, {"id": "end", "type": "end", "data": {}}, ], @@ -701,7 +668,7 @@ class WorkflowGraphTests(unittest.TestCase): "action": { "nodes": [ {"id": "start", "type": "start", "data": {}}, - {"id": "action", "type": "action", "data": {}}, + {"id": "action", "type": "action", "data": {"toolBinding": {"toolId": "tool"}}}, {"id": "agent", "type": "agent", "data": {}}, {"id": "handoff", "type": "handoff", "data": {}}, {"id": "end", "type": "end", "data": {}}, @@ -718,7 +685,7 @@ class WorkflowGraphTests(unittest.TestCase): {"id": "start", "type": "start", "data": {}}, {"id": "handoff", "type": "handoff", "data": {}}, {"id": "agent", "type": "agent", "data": {}}, - {"id": "action", "type": "action", "data": {}}, + {"id": "action", "type": "action", "data": {"toolBinding": {"toolId": "tool"}}}, {"id": "end", "type": "end", "data": {}}, ], "edges": [ @@ -785,7 +752,14 @@ class WorkflowGraphTests(unittest.TestCase): def test_rejects_automatic_cycle_and_duplicate_priorities(self): graph = valid_graph() - graph["nodes"].insert(1, {"id": "action", "type": "action", "data": {}}) + graph["nodes"].insert( + 1, + { + "id": "action", + "type": "action", + "data": {"toolBinding": {"toolId": "tool"}}, + }, + ) graph["edges"] = [ {"id": "a", "source": "start", "target": "action", "data": {"mode": "always", "priority": 0}}, {"id": "b", "source": "action", "target": "start", "data": {"mode": "always", "priority": 0}}, diff --git a/docs-developer/docs.json b/docs-developer/docs.json index 5476154..18107c7 100644 --- a/docs-developer/docs.json +++ b/docs-developer/docs.json @@ -46,7 +46,7 @@ "groups": [ { "group": "开始与对接", - "pages": ["index", "integrations", "realtime-protocol"] + "pages": ["index", "integrations", "tool-resources", "realtime-protocol"] } ] } diff --git a/docs-developer/tool-resources.mdx b/docs-developer/tool-resources.mdx new file mode 100644 index 0000000..c846195 --- /dev/null +++ b/docs-developer/tool-resources.mdx @@ -0,0 +1,98 @@ +--- +title: 工具资源体系 +description: 区分平台内建工具、可复用工具资源、使用绑定和运行时状态。 +icon: wrench +--- + +# 工具资源体系 + +工具配置分成三层。每一层只保存自己负责的信息，避免同一个工具在不同 Assistant 中无法复用。 + +| 层级 | API 字段 | 负责内容 | 例子 | +| --- | --- | --- | --- | +| 平台内建工具 | `builtInTools` | 与对话运行时紧密绑定的能力 | 结束对话、更新状态、跳过本轮、转接人工 | +| 工具资源 | `Tool` | 可复用的连接、请求/响应结构和工具私密 | HTTP、MCP、Client Tool | +| 工具绑定 | `toolBindings` / `toolBinding` | 某次使用中的参数来源、模板输入和结果映射 | `order_id` 由模型填写，`tenant` 来自变量 | + +## 平台内建工具 + +内建工具不是“组件库 / 工具资源”中的记录，也没有 `toolId`。Prompt 在 Assistant 层配置，Workflow 可在全局或独立 Agent 阶段配置。 + +```json +{ + "builtInTools": { + "endConversation": { + "description": "用户告别或任务完成时结束对话", + "messageType": "none", + "customMessage": "", + "captureReason": true, + "allowEndInterruption": false + } + } +} +``` + +`updateState` 可像 ElevenLabs Update State 一样修改已声明的动态变量。每条规则使用显式表达式，可由 `constant`、`variable`、`llm`、`if`、布尔/比较/算术节点组成。内建 `updateState` 允许 `llm` 叶子；Workflow 的确定性 Update State 节点复用同一表达式结构，但不允许 `llm` 叶子。 + +## 可复用工具资源 + +资源只保存所有使用者共享的内容： + +- HTTP：URL、method、headers、body 结构、参数 schema。 +- MCP：Server 引用、远端工具名和 input schema。 +- Client Tool：客户端函数名、参数 schema 和等待策略。 +- Secrets：仅在该资源执行时注入 HTTP Header，必须使用 `secret__` 前缀，不注册成全局对话变量。 + +需要不同 Assistant 提供不同值的占位符使用 `{{input__name}}`。资源声明 `templateInputs`，具体值由绑定提供。 + +## 工具绑定 + +```json +{ + "toolId": "tool_query_order", + "argumentBindings": { + "order_id": { "source": "model" }, + "tenant": { "source": "variable", "variableName": "tenant_id" } + }, + "templateBindings": { + "region": { "source": "literal", "value": "cn" } + }, + "resultAssignments": { + "order_status": "status" + } +} +``` + +参数来源有四种： + +- `model`：暴露给 LLM，由模型在调用时填写。 +- `variable`：读取已声明的动态变量或平台 `system__*` 变量。 +- `literal`：绑定中的固定值。 +- `template`：绑定中的模板值，可在执行时渲染普通动态变量。 + +Startup Action 和 Workflow Action 是确定性调用，不允许 `model` 来源。`resultAssignments` 的目标必须是已声明的动态变量。不等待响应的 Client Tool 不能配置结果映射。 + +## 动态变量还是 Client Tool + +选择标准是“状态存在哪里”： + +- 如果只是改变对话中的业务状态，使用内建 `updateState`。 +- 如果需要立即操作浏览器 UI、设备或宿主应用，使用 Client Tool。 + +因此 `set_photo_button_visible` 在当前语义下保持 Client Tool：它是对浏览器按钮的命令式副作用，不是只更新后端会话状态。如果未来 UI 完全以服务端状态声明式渲染，可改为更新 `photo_button_visible` 变量，由客户端订阅状态事件。 + +Client Tool 连接时会上报支持的函数名。当前客户端未上报某个函数时，Broker 在调用前立即返回错误，不等待超时。 + +## 版本边界 + +Tool Resource v4 是一次性迁移：公开 Workflow 写入接口只接受 `specVersion: 4`。数据库迁移会同时迁移 Prompt 和 Workflow，并删除原 System 工具资源。回滚该迁移需要使用部署前备份。 + +## Prompt Realtime 能力边界 + +Prompt Realtime 不再用运行模式做统一禁用，而是读取所选 Realtime 接口的 `fieldSchema.features.realtimePromptTools` 能力声明： + +- Qwen Audio Realtime 支持 HTTP、MCP、Client Tool，以及 `endConversation`、`updateState`、`requestHumanHandoff`。 +- `skipTurn` 只在 Pipeline 开放。端到端音频模型在工具调用前可能已经输出音频，无法保证绝对静默。 +- StepFun Prompt Realtime 当前没有声明工具能力，因此不能保存带工具的配置。Workflow Realtime 的既有行为不受这个 Prompt 门控影响。 + +切换运行模式或 Realtime Provider 不会自动删除工具配置。编辑器会显示不兼容项并提供显式清理操作；保存接口和会话启动都会再次校验，避免旧数据、inline config 或模型资源变更绕过能力限制。 diff --git a/frontend/src/components/assistant-editor/built-in-tools.tsx b/frontend/src/components/assistant-editor/built-in-tools.tsx new file mode 100644 index 0000000..d00d2bc --- /dev/null +++ b/frontend/src/components/assistant-editor/built-in-tools.tsx @@ -0,0 +1,205 @@ +"use client"; + +import { useState } from "react"; + +import { Input } from "@/components/ui/input"; +import { Switch } from "@/components/ui/switch"; +import { Textarea } from "@/components/ui/textarea"; +import type { BuiltInTools, DynamicVariableDefinition } from "@/lib/api"; + +const DEFAULTS = { + endConversation: { + description: "礼貌地结束本次对话。", + messageType: "none" as const, + customMessage: "", + captureReason: true, + allowEndInterruption: false, + }, + updateState: { description: "更新已声明的会话状态。", updates: [] }, + skipTurn: { description: "跳过当前轮次，不生成语音回复。" }, + requestHumanHandoff: { description: "提交人工接管请求。" }, +}; + +function UpdateRulesEditor({ + value, + onChange, +}: { + value: NonNullable["updates"]; + onChange: (value: NonNullable["updates"]) => void; +}) { + const [draft, setDraft] = useState(JSON.stringify(value, null, 2)); + const [error, setError] = useState(""); + + function commit() { + try { + const parsed = JSON.parse(draft) as unknown; + if (!Array.isArray(parsed)) throw new Error(); + setError(""); + onChange(parsed); + } catch { + setError("更新规则必须是 JSON 数组"); + } + } + + return ( +
+
+ 更新规则（JSON，支持 constant / variable / llm / if / 运算表达式） +
+