wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: organize tools and enable qwen realtime calls

Browse Source
This commit is contained in:
Xin Wang
2026-08-14 00:15:00 +08:00
parent eb111a3890
commit 5be21f5ccc
71 changed files with 5771 additions and 2026 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/schemas.py
+166 -46
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
from datetime import UTC, datetime
import re
from typing import Annotated, Any, Literal, Union
from typing import Annotated, Any, ClassVar, Literal, Union
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from pydantic.alias_generators import to_camel
@@ -20,7 +20,7 @@ ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent"]
AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"]
TurnEndStrategy = Literal["silence", "smart_turn"]
KnowledgeRetrievalMode = Literal["automatic", "on_demand"]
ToolType = Literal["system", "http", "mcp", "client"]
ToolType = Literal["http", "mcp", "client"]
ToolStatus = Literal["active", "archived", "draft"]
McpServerStatus = Literal["active", "archived", "draft"]
McpTransport = Literal["streamable_http", "sse"]
@@ -31,12 +31,7 @@ ClientToolResponseWaitMode = Literal["timeout", "session"]
DynamicVariableType = Literal["string", "number", "boolean"]
PromptEntryMode = Literal["wait_user", "generate"]
PromptOpeningMode = Literal["interruptible", "playback", "confirmation"]
SystemToolKind = Literal[
"end_conversation",
"update_state",
"skip_turn",
"request_human_handoff",
]
PrimitiveValue = str | int | float | bool
# 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵
EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"}
@@ -140,11 +135,55 @@ ALLOWED_FIELDS: dict[str, set[str]] = {
}
class ModelValueSource(CamelModel):
source: Literal["model"] = "model"
class VariableValueSource(CamelModel):
source: Literal["variable"] = "variable"
variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
class LiteralValueSource(CamelModel):
source: Literal["literal"] = "literal"
value: Any = None
class TemplateValueSource(CamelModel):
source: Literal["template"] = "template"
value: Any
ValueSource = Annotated[
Union[
ModelValueSource,
VariableValueSource,
LiteralValueSource,
TemplateValueSource,
],
Field(discriminator="source"),
]
NonModelValueSource = Annotated[
Union[VariableValueSource, LiteralValueSource, TemplateValueSource],
Field(discriminator="source"),
]
class ToolBinding(CamelModel):
"""One assistant/stage-specific use of a reusable external tool."""
tool_id: str = Field(min_length=1, max_length=40)
argument_bindings: dict[str, ValueSource] = Field(default_factory=dict)
template_bindings: dict[str, NonModelValueSource] = Field(default_factory=dict)
result_assignments: dict[str, str] = Field(default_factory=dict)
class StartupAction(CamelModel):
id: str = Field(min_length=1, max_length=64, pattern=r"^[A-Za-z][A-Za-z0-9_-]*$")
phase: Literal["preflight", "opening"] = "opening"
tool_id: str = Field(min_length=1, max_length=40)
arguments: dict[str, Any] = Field(default_factory=dict)
tool_binding: ToolBinding
required: bool = True
@@ -190,6 +229,11 @@ class StartupConfig(CamelModel):
# ---------- 助手(单表 STI:瘦类型真列 + workflow 图 JSON 列) ----------
class AssistantUpsert(CamelModel):
# AssistantOut inherits this DTO for a stable public shape, but persisted
# historical rows must remain readable. Only request models enforce
# runtime combinations that the current editor/runtime can create.
enforce_request_constraints: ClassVar[bool] = True
name: str
type: AssistantType = "prompt"
runtime_mode: RuntimeMode = "pipeline"
@@ -206,7 +250,8 @@ class AssistantUpsert(CamelModel):
knowledge_retrieval_config: KnowledgeRetrievalConfig = Field(
default_factory=KnowledgeRetrievalConfig
)
tool_ids: list[str] = Field(default_factory=list)
tool_bindings: list[ToolBinding] = Field(default_factory=list)
built_in_tools: "BuiltInTools" = Field(default_factory=lambda: BuiltInTools())
# 瘦类型专属(真列);按 type 取用,无关字段写入时清零
prompt: str = ""
@@ -244,17 +289,24 @@ class AssistantUpsert(CamelModel):
if self.type == "workflow":
self.greeting = ""
if self.type not in {"prompt", "workflow"}:
self.tool_ids = []
self.tool_bindings = []
self.built_in_tools = BuiltInTools()
self.dynamic_variable_definitions = {}
if self.type != "prompt":
self.startup = StartupConfig()
if self.runtime_mode == "realtime" and (
self.startup.actions
or self.startup.opening_message is not None
or self.startup.opening_mode != "interruptible"
or self.startup.entry_mode != "wait_user"
):
raise ValueError("Prompt Realtime 模式暂不支持启动阶段")
if self.type == "workflow":
# Workflow usage configuration lives inside graph v4. Top-level
# bindings remain a Prompt-only public API.
self.tool_bindings = []
self.built_in_tools = BuiltInTools()
if self.enforce_request_constraints:
if self.runtime_mode == "realtime" and (
self.startup.actions
or self.startup.opening_message is not None
or self.startup.opening_mode != "interruptible"
or self.startup.entry_mode != "wait_user"
):
raise ValueError("Prompt Realtime 模式暂不支持启动阶段")
# 外部托管大脑只能 cascade,拦住不兼容的 realtime
if self.runtime_mode == "realtime" and self.type not in REALTIME_CAPABLE_TYPES:
raise ValueError(f"类型 {self.type} 不支持 realtime 运行模式")
@@ -262,6 +314,8 @@ class AssistantUpsert(CamelModel):
class AssistantOut(AssistantUpsert):
enforce_request_constraints: ClassVar[bool] = False
id: str
updated_at: str | None = None
@@ -289,6 +343,94 @@ class DynamicVariableDefinition(CamelModel):
return self
# ---------- 平台内建工具 ----------
class ConstantStateExpression(CamelModel):
type: Literal["constant"] = "constant"
value: PrimitiveValue
class VariableStateExpression(CamelModel):
type: Literal["variable"] = "variable"
variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
class LlmStateExpression(CamelModel):
type: Literal["llm"] = "llm"
parameter_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
value_type: DynamicVariableType = "string"
description: str = Field(min_length=1, max_length=500)
class OperationStateExpression(CamelModel):
type: Literal[
"and",
"or",
"eq",
"neq",
"gt",
"gte",
"lt",
"lte",
"add",
"subtract",
"multiply",
"divide",
]
operands: list["StateExpression"] = Field(min_length=2, max_length=10)
class ConditionalStateExpression(CamelModel):
type: Literal["if"] = "if"
condition: "StateExpression"
then: "StateExpression"
otherwise: "StateExpression"
StateExpression = Annotated[
Union[
ConstantStateExpression,
VariableStateExpression,
LlmStateExpression,
OperationStateExpression,
ConditionalStateExpression,
],
Field(discriminator="type"),
]
class StateUpdateRule(CamelModel):
variable_name: str = Field(pattern=r"^[A-Za-z][A-Za-z0-9_]{0,63}$")
expression: StateExpression
class BuiltInToolBase(CamelModel):
description: str = Field(default="", max_length=2048)
class EndConversationTool(BuiltInToolBase):
message_type: Literal["none", "custom"] = "none"
custom_message: str = Field(default="", max_length=2000)
capture_reason: bool = True
allow_end_interruption: bool = False
class UpdateStateTool(BuiltInToolBase):
updates: list[StateUpdateRule] = Field(default_factory=list, max_length=10)
class BuiltInTools(CamelModel):
end_conversation: EndConversationTool | None = None
update_state: UpdateStateTool | None = None
skip_turn: BuiltInToolBase | None = None
request_human_handoff: BuiltInToolBase | None = None
OperationStateExpression.model_rebuild()
ConditionalStateExpression.model_rebuild()
AssistantUpsert.model_rebuild()
AssistantOut.model_rebuild()
# ---------- 可复用工具 ----------
class ToolParameter(CamelModel):
name: str
@@ -298,14 +440,6 @@ class ToolParameter(CamelModel):
required: bool = True
class SystemToolConfig(CamelModel):
kind: SystemToolKind = "end_conversation"
message_type: Literal["none", "custom"] = "none"
custom_message: str = ""
capture_reason: bool = True
allow_end_interruption: bool = False
class HttpToolConfig(CamelModel):
allow_interruptions: bool = True
execution_mode: ToolExecutionMode = "immediate"
@@ -314,8 +448,8 @@ class HttpToolConfig(CamelModel):
timeout_seconds: int = Field(default=15, ge=1, le=120)
headers: dict[str, str] = Field(default_factory=dict)
parameters: list[ToolParameter] = Field(default_factory=list)
template_inputs: list[ToolParameter] = Field(default_factory=list)
body: dict[str, Any] = Field(default_factory=dict)
dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict)
@field_validator("url")
@classmethod
@@ -325,14 +459,8 @@ class HttpToolConfig(CamelModel):
return value
class SystemToolDefinition(CamelModel):
schema_version: int = 1
type: Literal["system"] = "system"
config: SystemToolConfig = Field(default_factory=SystemToolConfig)
class HttpToolDefinition(CamelModel):
schema_version: int = 1
schema_version: int = 2
type: Literal["http"] = "http"
config: HttpToolConfig
@@ -344,17 +472,10 @@ class ClientToolConfig(CamelModel):
response_wait_mode: ClientToolResponseWaitMode = "timeout"
parameters: list[ToolParameter] = Field(default_factory=list)
timeout_seconds: int = Field(default=3, ge=1, le=30)
dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict)
@model_validator(mode="after")
def validate_result_assignments(self):
if not self.wait_for_response and self.dynamic_variable_assignments:
raise ValueError("Client Tool 不等待响应时不能配置结果变量赋值")
return self
class ClientToolDefinition(CamelModel):
schema_version: int = 1
schema_version: int = 2
type: Literal["client"] = "client"
config: ClientToolConfig = Field(default_factory=ClientToolConfig)
@@ -365,18 +486,17 @@ class McpToolConfig(CamelModel):
remote_tool_name: str = Field(min_length=1, max_length=255)
input_schema: dict[str, Any] = Field(default_factory=dict)
schema_hash: str = ""
dynamic_variable_assignments: dict[str, str] = Field(default_factory=dict)
template_inputs: list[ToolParameter] = Field(default_factory=list)
class McpToolDefinition(CamelModel):
schema_version: int = 1
schema_version: int = 2
type: Literal["mcp"] = "mcp"
config: McpToolConfig
ToolDefinition = Annotated[
Union[
SystemToolDefinition,
HttpToolDefinition,
ClientToolDefinition,
McpToolDefinition,