wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: support vision prompt modes

Browse Source
This commit is contained in:
Xin Wang
2026-08-12 19:13:04 +08:00
parent 09b07751b6
commit 94a02d7e2b
29 changed files with 676 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/services/brains/workflow_brain.py
+4 -1
View File
@@ -670,7 +670,10 @@ class WorkflowBrain(BaseBrain):
append_function(self._flow_tool(tool, node_id))
append_function(self._knowledge_function(node_id))
if stage.vision_enabled and self._require_runtime().vision_function:
append_function(self._require_runtime().vision_function)
vision_function = self._require_runtime().vision_function
append_function(
vision_function(stage) if callable(vision_function) else vision_function
)
if self._engine.llm_routing_mode() == "edge_tool":
for edge in self._engine.edge_tool_edges(node_id):
append_function(self._edge_tool(edge, node_id))
backend/services/config_resolver.py
+2 -1
View File
@@ -301,7 +301,8 @@ async def resolve_runtime_config(
),
vision_enabled=assistant.vision_enabled,
vision_model_resource_id=assistant.vision_model_resource_id,
vision_focus_prompt=assistant.vision_focus_prompt or "",
vision_prompt_mode=assistant.vision_prompt_mode or "runtime",
vision_prompt=assistant.vision_prompt or "",
vision_model=str(_value(vision_resource, "modelId", "")),
vision_llm_interface_type=(
vision_resource.interface_type if vision_resource else "openai-llm"
backend/services/node_specs.py
+25 -3
View File
@@ -6,6 +6,7 @@ from collections import defaultdict, deque
from copy import deepcopy
from typing import Any
from models import VISION_PROMPT_MODES
from services.message_policy import (
MESSAGE_COMPLETION_POLICIES,
MESSAGE_CONFIRMATION,
@@ -199,6 +200,11 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
data["stateVariableNames"] = list(
dict.fromkeys(str(name) for name in state_names or [] if str(name))
)
legacy_prompt = str(data.pop("visionFocusPrompt", "") or "").strip()
if "visionPrompt" not in data:
data["visionPrompt"] = legacy_prompt
if "visionPromptMode" not in data:
data["visionPromptMode"] = "guided" if legacy_prompt else "runtime"
if "inheritGlobalConfig" not in data:
has_node_overrides = any(
(
@@ -207,7 +213,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
data.get("ttsResourceId"),
data.get("visionEnabled"),
data.get("visionModelResourceId"),
data.get("visionFocusPrompt"),
data.get("visionPrompt"),
data.get("knowledgeBaseId"),
data.get("toolIds"),
)
@@ -268,7 +274,10 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
settings.setdefault("defaultTtsResourceId", "")
settings.setdefault("visionEnabled", False)
settings.setdefault("visionModelResourceId", "")
settings.setdefault("visionFocusPrompt", "")
legacy_prompt = str(settings.pop("visionFocusPrompt", "") or "").strip()
settings.setdefault("visionPrompt", legacy_prompt)
if "visionPromptMode" not in settings:
settings["visionPromptMode"] = "guided" if legacy_prompt else "runtime"
settings.setdefault("toolIds", [])
settings.setdefault("knowledgeBaseId", "")
settings.setdefault("knowledgeMode", "automatic")
@@ -471,6 +480,20 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
not isinstance(name, str) for name in state_names
):
errors.append(f"Agent 节点 {node_id} 的状态变量授权必须是列表")
source = settings if data.get("inheritGlobalConfig", True) else data
vision_prompt_mode = str(source.get("visionPromptMode") or "runtime")
if vision_prompt_mode not in VISION_PROMPT_MODES:
errors.append(
f"Agent 节点 {node_id} 的视觉提示词模式无效:{vision_prompt_mode}"
)
if (
source.get("visionEnabled")
and vision_prompt_mode in {"fixed", "guided"}
and not str(source.get("visionPrompt") or "").strip()
):
errors.append(
f"Agent 节点 {node_id} 的固定或引导模式必须填写视觉提示词"
)
if runtime_mode == "realtime":
if data.get("contextPolicy", "inherit") != "inherit":
errors.append(
@@ -487,7 +510,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
errors.append(
f"Realtime Agent 节点 {node_id} 不支持节点级模型、语音或交互策略覆盖"
)
source = settings if data.get("inheritGlobalConfig", True) else data
if (
source.get("knowledgeBaseId")
and source.get("knowledgeMode") != "on_demand"
backend/services/pipecat/pipeline.py
+61 -25
View File
@@ -38,8 +38,9 @@ from services.vision import (
image_data_uri,
image_frame_from_jpeg,
image_jpeg_bytes,
resolve_vision_question,
vision_system_hint,
vision_tool_description,
vision_tool_schema_parts,
)
from services.workflow_engine import WorkflowEngine
@@ -349,7 +350,9 @@ async def run_pipeline(
def with_vision_hint(text: str) -> str:
hints = []
if vision_enabled and cfg.type != "workflow":
hints.append(vision_system_hint(cfg.vision_focus_prompt))
hints.append(
vision_system_hint(cfg.vision_prompt_mode, cfg.vision_prompt)
)
if cfg.knowledge_base_id:
hints.append(
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
@@ -450,7 +453,8 @@ async def run_pipeline(
workflow_vision_scope: dict[str, Any] = {
"enabled": False,
"vision_model_resource_id": None,
"vision_focus_prompt": "",
"vision_prompt_mode": "runtime",
"vision_prompt": "",
"llm_resource_id": None,
}
@@ -458,16 +462,15 @@ async def run_pipeline(
if cfg.type == "workflow":
return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope)
return vision_native_mode
vision_description, vision_properties, vision_required = vision_tool_schema_parts(
cfg.vision_prompt_mode,
cfg.vision_prompt,
)
vision_schema = FunctionSchema(
name=VISION_TOOL_NAME,
description=vision_tool_description(cfg.vision_focus_prompt),
properties={
"question": {
"type": "string",
"description": "用户关于当前视频画面的具体问题。",
}
},
required=["question"],
description=vision_description,
properties=vision_properties,
required=vision_required,
)
knowledge_schema = FunctionSchema(
name=KNOWLEDGE_TOOL_NAME,
@@ -498,7 +501,15 @@ async def run_pipeline(
await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"})
async def fetch_user_image(params: FunctionCallParams):
question = str(params.arguments.get("question") or "请描述当前画面。")
try:
question = resolve_vision_question(
cfg.vision_prompt_mode,
cfg.vision_prompt,
params.arguments,
)
except ValueError as exc:
await params.result_callback({"status": "error", "message": str(exc)})
return
user_id = vision_state.get("client_id")
if not user_id:
await params.result_callback(
@@ -576,14 +587,26 @@ async def run_pipeline(
result = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
result.vision_focus_prompt = str(
workflow_vision_scope.get("vision_focus_prompt") or ""
result.vision_prompt_mode = str(
workflow_vision_scope.get("vision_prompt_mode") or "runtime"
)
result.vision_prompt = str(
workflow_vision_scope.get("vision_prompt") or ""
).strip()
return result
if cfg.type == "workflow" and vision_enabled:
async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:
question = str(params.arguments.get("question") or "请描述当前画面。")
try:
vision_cfg = active_vision_config()
question = resolve_vision_question(
vision_cfg.vision_prompt_mode,
vision_cfg.vision_prompt,
params.arguments,
)
except ValueError as exc:
await params.result_callback({"status": "error", "message": str(exc)})
return
user_id = vision_state.get("client_id")
if not user_id:
await params.result_callback(
@@ -609,7 +632,15 @@ async def run_pipeline(
"status": "disabled",
"message": "当前 Agent 节点未开启视觉理解。",
}
question = str((args or {}).get("question") or "请描述当前画面。")
vision_cfg = active_vision_config()
try:
question = resolve_vision_question(
vision_cfg.vision_prompt_mode,
vision_cfg.vision_prompt,
args,
)
except ValueError as exc:
return {"status": "error", "message": str(exc)}
user_id = vision_state.get("client_id")
if not user_id:
return {
@@ -625,7 +656,7 @@ async def run_pipeline(
try:
frame = await vision_capture.request_image(llm, request)
observation = await analyze_image_with_vision_model(
active_vision_config(),
vision_cfg,
frame,
question,
)
@@ -654,14 +685,19 @@ async def run_pipeline(
active_vision_uses_main_llm,
)
workflow_vision_function = FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=vision_schema.description,
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
def workflow_vision_function(stage) -> FlowsFunctionSchema:
description, properties, required = vision_tool_schema_parts(
stage.vision_prompt_mode,
stage.vision_prompt,
)
return FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=description,
properties=properties,
required=required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
if vision_enabled and cfg.type != "workflow":
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
backend/services/vision.py
+85 -74
View File
@@ -5,8 +5,14 @@ from __future__ import annotations
import asyncio
import base64
from io import BytesIO
from typing import Any
from models import AssistantConfig, RuntimeModelResource
from models import (
VISION_PROMPT_MODES,
AssistantConfig,
RuntimeModelResource,
VisionPromptMode,
)
from openai import AsyncOpenAI
from PIL import Image
from pipecat.frames.frames import UserImageRawFrame
@@ -22,12 +28,6 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
"如果画面不足以判断，请明确说明不确定。"
)
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
"不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
"意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
)
def _require(value: str, label: str) -> str:
@@ -36,83 +36,96 @@ def _require(value: str, label: str) -> str:
raise ValueError(f"缺少模型资源配置: {label}")
def vision_system_hint(focus_prompt: str = "") -> str:
"""Describe when to use vision and keep a configured business focus visible."""
def normalize_vision_prompt_mode(value: object) -> VisionPromptMode:
"""Return a supported mode while keeping older saved configs compatible."""
focus = focus_prompt.strip()
if not focus:
return VISION_SYSTEM_HINT
mode = str(value or "runtime")
if mode == "fixed":
return "fixed"
if mode == "guided":
return "guided"
return "runtime"
def _compact_prompt(prompt: str, *, limit: int = 4000) -> str:
return " ".join(prompt.split())[:limit]
def vision_system_hint(
mode: VisionPromptMode = "runtime",
configured_prompt: str = "",
) -> str:
"""Tell the conversation model how the camera tool forms its question."""
prompt_mode = normalize_vision_prompt_mode(mode)
prompt = _compact_prompt(configured_prompt)
if prompt_mode == "fixed":
return (
f"{VISION_SYSTEM_HINT}\n"
f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。"
)
if prompt_mode == "guided":
return (
f"{VISION_SYSTEM_HINT}\n"
"调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导："
f"{prompt}"
)
return (
f"{VISION_SYSTEM_HINT}\n"
f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
"调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。"
)
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
"""Keep the safety baseline fixed while appending an editable business focus."""
def vision_tool_schema_parts(
mode: VisionPromptMode = "runtime",
configured_prompt: str = "",
) -> tuple[str, dict[str, Any], list[str]]:
"""Build the tool contract for one configured visual-question mode."""
focus = focus_prompt.strip()
if not focus:
return VISION_ANALYSIS_SYSTEM_PROMPT
return (
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
f"[业务视觉关注目标]\n{focus}\n"
"该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
)
def vision_tool_description(focus_prompt: str = "") -> str:
base = (
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
"人/物品/环境状态或需要视觉判断时调用。"
)
focus = " ".join(focus_prompt.split())
return f"{base} 当前业务重点：{focus[:600]}" if focus else base
async def generate_vision_focus_prompt(
resource: RuntimeModelResource,
source_prompt: str,
) -> str:
"""Generate one editable focus draft with an existing LLM resource."""
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
source = source_prompt.strip()
if not source:
raise ValueError("助手提示词不能为空")
values = resource.values or {}
secrets = resource.secrets or {}
extra_body = values.get("extraBody")
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
client = AsyncOpenAI(
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
)
try:
response = await client.chat.completions.create(
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
messages=[
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
{"role": "user", "content": source},
],
**extra,
prompt_mode = normalize_vision_prompt_mode(mode)
prompt = _compact_prompt(configured_prompt, limit=600)
if prompt_mode == "fixed":
return (
f"{base} 每次都检查同一个固定问题：{prompt}",
{},
[],
)
finally:
await client.close()
content = response.choices[0].message.content if response.choices else ""
result = (
content.strip()
if isinstance(content, str)
else str(content or "").strip()
question_description = "结合当前对话生成的、关于当前视频画面的具体问题。"
description = base
if prompt_mode == "guided":
description = f"{base} 生成问题时必须遵守固定引导：{prompt}"
question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}"
return (
description,
{"question": {"type": "string", "description": question_description}},
["question"],
)
if not result:
raise ValueError("模型没有生成有效的视觉关注目标")
if result.startswith("```") and result.endswith("```"):
result = result[3:-3].strip()
if result.startswith("text"):
result = result[4:].strip()
return result[:4000]
def resolve_vision_question(
mode: VisionPromptMode,
configured_prompt: str,
arguments: dict[str, Any] | None,
) -> str:
"""Resolve the exact question sent with the captured image."""
prompt_mode = normalize_vision_prompt_mode(mode)
if prompt_mode == "fixed":
question = configured_prompt.strip()
else:
question = str((arguments or {}).get("question") or "").strip()
if not question:
raise ValueError(
"固定视觉提示词为空"
if prompt_mode == "fixed"
else "视觉问题为空"
)
return question
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
@@ -173,9 +186,7 @@ async def analyze_image_with_vision_model(
messages=[
{
"role": "system",
"content": vision_analysis_system_prompt(
cfg.vision_focus_prompt
),
"content": VISION_ANALYSIS_SYSTEM_PROMPT,
},
{
"role": "user",
backend/services/workflow/agent.py
+3 -2
View File
@@ -49,7 +49,7 @@ class WorkflowAgentStage:
if stage.vision_enabled:
stage_prompt = (
f"{stage_prompt}\n\n[视觉能力]\n"
f"{vision_system_hint(stage.vision_focus_prompt)}"
f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}"
)
instruction = (
EDGE_TOOL_STAGE_INSTRUCTION
@@ -112,7 +112,8 @@ class WorkflowAgentStage:
{
"enabled": stage.vision_enabled,
"vision_model_resource_id": stage.vision_model_resource_id,
"vision_focus_prompt": stage.vision_focus_prompt,
"vision_prompt_mode": stage.vision_prompt_mode,
"vision_prompt": stage.vision_prompt,
"llm_resource_id": stage.llm_resource_id,
}
)
backend/services/workflow/realtime.py
+21 -14
View File
@@ -34,8 +34,9 @@ from services.vision import (
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_vision_resource,
resolve_vision_question,
vision_system_hint,
vision_tool_description,
vision_tool_schema_parts,
)
from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
@@ -414,7 +415,10 @@ class WorkflowRealtimeController:
f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
)
if stage.vision_enabled:
prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
prompt = (
f"{prompt}\n"
f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}"
)
return prompt
def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
@@ -573,10 +577,15 @@ class WorkflowRealtimeController:
{"status": "stale", "message": "当前 Agent 已经切换。"},
continue_response=False,
)
question = str(arguments.get("question") or "").strip()
if not question:
try:
question = resolve_vision_question(
stage.vision_prompt_mode,
stage.vision_prompt,
arguments,
)
except ValueError as exc:
return RealtimeToolResult(
{"status": "error", "message": "视觉问题为空"}
{"status": "error", "message": str(exc)}
)
if self._runtime.capture_image is None:
return RealtimeToolResult(
@@ -605,7 +614,6 @@ class WorkflowRealtimeController:
try:
if vision_cfg is not None:
vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
observation = await analyze_image_with_vision_model(
vision_cfg,
frame,
@@ -634,17 +642,16 @@ class WorkflowRealtimeController:
}
)
description, properties, required = vision_tool_schema_parts(
stage.vision_prompt_mode,
stage.vision_prompt,
)
return (
RealtimeTool(
name=VISION_TOOL_NAME,
description=vision_tool_description(stage.vision_focus_prompt),
properties={
"question": {
"type": "string",
"description": "需要根据当前画面判断的具体问题",
}
},
required=("question",),
description=description,
properties=properties,
required=tuple(required),
),
handler,
)
backend/services/workflow_engine.py
+4 -2
View File
@@ -23,7 +23,8 @@ class AgentStageConfig:
tts_resource_id: str
vision_enabled: bool
vision_model_resource_id: str | None
vision_focus_prompt: str
vision_prompt_mode: str
vision_prompt: str
tool_ids: tuple[str, ...]
state_variable_names: tuple[str, ...]
knowledge_base_id: str | None
@@ -152,7 +153,8 @@ class WorkflowEngine:
vision_model_resource_id=(
str(source.get("visionModelResourceId") or "") or None
),
vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(),
vision_prompt_mode=str(source.get("visionPromptMode") or "runtime"),
vision_prompt=str(source.get("visionPrompt") or "").strip(),
tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []),
# Variable write scope always belongs to the Agent node, even when
# its selected tool resources come from Workflow defaults.