wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions 3 Packages Projects Releases Wiki Activity

fix: support vision prompt modes

Browse Source
This commit is contained in:
Xin Wang
2026-08-12 19:13:04 +08:00
parent 09b07751b6
commit 94a02d7e2b
29 changed files with 676 additions and 439 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/db/models.py
+2 -1
View File
@@ -160,7 +160,8 @@ class Assistant(Base):
ForeignKey("model_resources.id", ondelete="SET NULL"),
nullable=True,
)
vision_focus_prompt: Mapped[str] = mapped_column(Text, default="")
vision_prompt_mode: Mapped[str] = mapped_column(String(16), default="runtime")
vision_prompt: Mapped[str] = mapped_column(Text, default="")
# KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
knowledge_base_id: Mapped[str | None] = mapped_column(
backend/migrations/versions/20260812_0020_add_vision_prompt_mode.py
+47
View File
@@ -0,0 +1,47 @@
"""replace vision focus with explicit prompt modes
Revision ID: 20260812_0020
Revises: 20260812_0019
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
revision: str = "20260812_0020"
down_revision: str | Sequence[str] | None = "20260812_0019"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.alter_column(
"assistants",
"vision_focus_prompt",
new_column_name="vision_prompt",
)
op.add_column(
"assistants",
sa.Column(
"vision_prompt_mode",
sa.String(length=16),
server_default="runtime",
nullable=False,
),
)
# The previous single-field implementation behaved most like guided mode.
op.execute(
"UPDATE assistants SET vision_prompt_mode = 'guided' "
"WHERE TRIM(vision_prompt) <> ''"
)
def downgrade() -> None:
op.drop_column("assistants", "vision_prompt_mode")
op.alter_column(
"assistants",
"vision_prompt",
new_column_name="vision_focus_prompt",
)
backend/models.py
+4 -1
View File
@@ -13,6 +13,8 @@ from typing import Literal
from pydantic import BaseModel, Field
RuntimeMode = Literal["pipeline", "realtime"]
VisionPromptMode = Literal["fixed", "runtime", "guided"]
VISION_PROMPT_MODES = frozenset({"fixed", "runtime", "guided"})
class RuntimeMcpServer(BaseModel):
@@ -92,7 +94,8 @@ class AssistantConfig(BaseModel):
llm_support_image_input: bool = False
vision_enabled: bool = False
vision_model_resource_id: str | None = None
vision_focus_prompt: str = ""
vision_prompt_mode: VisionPromptMode = "runtime"
vision_prompt: str = ""
vision_model: str = ""
vision_llm_interface_type: str = "openai-llm"
vision_llm_values: dict = {}
backend/routes/assistants.py
+14 -44
View File
@@ -2,7 +2,7 @@
import uuid
from models import RuntimeModelResource, RuntimeTool
from models import VISION_PROMPT_MODES, RuntimeTool
from db.models import (
Assistant,
AssistantModelBinding,
@@ -16,15 +16,12 @@ from fastapi import APIRouter, Depends, HTTPException
from schemas import (
AssistantOut,
AssistantUpsert,
VisionFocusPromptGenerateRequest,
VisionFocusPromptGenerateResult,
)
from services.auth import require_admin
from services.masking import mask, resolve_incoming_key
from services.node_specs import graph_references, normalize_graph, validate_graph
from services.system_tools import system_tool_kind
from services.tool_policy import policy_for_tool
from services.vision import generate_vision_focus_prompt
from services.workflow_engine import WorkflowEngine
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
@@ -79,7 +76,8 @@ def _validate_workflow(body: AssistantUpsert) -> None:
# capability summary used by clients to decide whether to open the camera.
body.vision_enabled = WorkflowEngine(body.graph).uses_vision()
body.vision_model_resource_id = None
body.vision_focus_prompt = ""
body.vision_prompt_mode = "runtime"
body.vision_prompt = ""
refs = graph_references(body.graph)
body.tool_ids = list(dict.fromkeys([*body.tool_ids, *sorted(refs["tools"])]))
@@ -271,6 +269,13 @@ async def _validate_vision_model(
if body.type == "workflow":
return
if body.vision_enabled:
if body.vision_prompt_mode not in VISION_PROMPT_MODES:
raise HTTPException(400, "视觉提示词模式无效")
if (
body.vision_prompt_mode in {"fixed", "guided"}
and not body.vision_prompt.strip()
):
raise HTTPException(400, "固定或引导模式必须填写视觉提示词")
if body.vision_model_resource_id:
resource = await session.get(ModelResource, body.vision_model_resource_id)
else:
@@ -396,7 +401,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
startup=assistant.startup or {},
vision_enabled=assistant.vision_enabled,
vision_model_resource_id=assistant.vision_model_resource_id,
vision_focus_prompt=assistant.vision_focus_prompt or "",
vision_prompt_mode=assistant.vision_prompt_mode or "runtime",
vision_prompt=assistant.vision_prompt or "",
model_resource_ids=await _resource_ids(session, assistant.id),
knowledge_base_id=assistant.knowledge_base_id,
knowledge_retrieval_config=assistant.knowledge_retrieval_config or {},
@@ -448,43 +454,6 @@ async def create_assistant(
return await _to_out(session, assistant)
@router.post(
"/vision-focus/generate",
response_model=VisionFocusPromptGenerateResult,
)
async def generate_assistant_vision_focus(
body: VisionFocusPromptGenerateRequest,
session: AsyncSession = Depends(get_session),
):
"""Generate an editable visual focus draft from an assistant prompt."""
resource = await session.get(ModelResource, body.model_resource_id)
if (
resource is None
or resource.capability != "LLM"
or not resource.enabled
):
raise HTTPException(400, "请选择可用的大语言模型生成视觉关注目标")
try:
focus_prompt = await generate_vision_focus_prompt(
RuntimeModelResource(
id=resource.id,
name=resource.name,
capability=resource.capability,
interface_type=resource.interface_type,
values=resource.values or {},
secrets=resource.secrets or {},
support_image_input=bool(resource.support_image_input),
),
body.source_prompt,
)
except ValueError as exc:
raise HTTPException(400, str(exc)) from exc
except Exception as exc:
raise HTTPException(502, "生成视觉关注目标失败，请稍后重试") from exc
return VisionFocusPromptGenerateResult(focus_prompt=focus_prompt)
@router.get("/{assistant_id}", response_model=AssistantOut)
async def get_assistant(
assistant_id: str, session: AsyncSession = Depends(get_session)
@@ -513,7 +482,8 @@ async def duplicate_assistant(
startup=dict(source.startup or {}),
vision_enabled=source.vision_enabled,
vision_model_resource_id=source.vision_model_resource_id,
vision_focus_prompt=source.vision_focus_prompt or "",
vision_prompt_mode=source.vision_prompt_mode or "runtime",
vision_prompt=source.vision_prompt or "",
knowledge_base_id=source.knowledge_base_id,
knowledge_retrieval_config=dict(source.knowledge_retrieval_config or {}),
prompt=source.prompt,
backend/schemas.py
+3 -10
View File
@@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
from pydantic.alias_generators import to_camel
RuntimeMode = Literal["pipeline", "realtime"]
VisionPromptMode = Literal["fixed", "runtime", "guided"]
ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent"]
AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"]
TurnEndStrategy = Literal["silence", "smart_turn"]
@@ -198,7 +199,8 @@ class AssistantUpsert(CamelModel):
startup: StartupConfig = Field(default_factory=StartupConfig)
vision_enabled: bool = False
vision_model_resource_id: str | None = None
vision_focus_prompt: str = Field(default="", max_length=4000)
vision_prompt_mode: VisionPromptMode = "runtime"
vision_prompt: str = Field(default="", max_length=4000)
model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
knowledge_base_id: str | None = None
knowledge_retrieval_config: KnowledgeRetrievalConfig = Field(
@@ -264,15 +266,6 @@ class AssistantOut(AssistantUpsert):
updated_at: str | None = None
class VisionFocusPromptGenerateRequest(CamelModel):
source_prompt: str = Field(min_length=1, max_length=20000)
model_resource_id: str = Field(min_length=1, max_length=40)
class VisionFocusPromptGenerateResult(CamelModel):
focus_prompt: str
class DynamicVariableDefinition(CamelModel):
type: DynamicVariableType = "string"
required: bool = False
backend/services/brains/workflow_brain.py
+4 -1
View File
@@ -670,7 +670,10 @@ class WorkflowBrain(BaseBrain):
append_function(self._flow_tool(tool, node_id))
append_function(self._knowledge_function(node_id))
if stage.vision_enabled and self._require_runtime().vision_function:
append_function(self._require_runtime().vision_function)
vision_function = self._require_runtime().vision_function
append_function(
vision_function(stage) if callable(vision_function) else vision_function
)
if self._engine.llm_routing_mode() == "edge_tool":
for edge in self._engine.edge_tool_edges(node_id):
append_function(self._edge_tool(edge, node_id))
backend/services/config_resolver.py
+2 -1
View File
@@ -301,7 +301,8 @@ async def resolve_runtime_config(
),
vision_enabled=assistant.vision_enabled,
vision_model_resource_id=assistant.vision_model_resource_id,
vision_focus_prompt=assistant.vision_focus_prompt or "",
vision_prompt_mode=assistant.vision_prompt_mode or "runtime",
vision_prompt=assistant.vision_prompt or "",
vision_model=str(_value(vision_resource, "modelId", "")),
vision_llm_interface_type=(
vision_resource.interface_type if vision_resource else "openai-llm"
backend/services/node_specs.py
+25 -3
View File
@@ -6,6 +6,7 @@ from collections import defaultdict, deque
from copy import deepcopy
from typing import Any
from models import VISION_PROMPT_MODES
from services.message_policy import (
MESSAGE_COMPLETION_POLICIES,
MESSAGE_CONFIRMATION,
@@ -199,6 +200,11 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
data["stateVariableNames"] = list(
dict.fromkeys(str(name) for name in state_names or [] if str(name))
)
legacy_prompt = str(data.pop("visionFocusPrompt", "") or "").strip()
if "visionPrompt" not in data:
data["visionPrompt"] = legacy_prompt
if "visionPromptMode" not in data:
data["visionPromptMode"] = "guided" if legacy_prompt else "runtime"
if "inheritGlobalConfig" not in data:
has_node_overrides = any(
(
@@ -207,7 +213,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
data.get("ttsResourceId"),
data.get("visionEnabled"),
data.get("visionModelResourceId"),
data.get("visionFocusPrompt"),
data.get("visionPrompt"),
data.get("knowledgeBaseId"),
data.get("toolIds"),
)
@@ -268,7 +274,10 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
settings.setdefault("defaultTtsResourceId", "")
settings.setdefault("visionEnabled", False)
settings.setdefault("visionModelResourceId", "")
settings.setdefault("visionFocusPrompt", "")
legacy_prompt = str(settings.pop("visionFocusPrompt", "") or "").strip()
settings.setdefault("visionPrompt", legacy_prompt)
if "visionPromptMode" not in settings:
settings["visionPromptMode"] = "guided" if legacy_prompt else "runtime"
settings.setdefault("toolIds", [])
settings.setdefault("knowledgeBaseId", "")
settings.setdefault("knowledgeMode", "automatic")
@@ -471,6 +480,20 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
not isinstance(name, str) for name in state_names
):
errors.append(f"Agent 节点 {node_id} 的状态变量授权必须是列表")
source = settings if data.get("inheritGlobalConfig", True) else data
vision_prompt_mode = str(source.get("visionPromptMode") or "runtime")
if vision_prompt_mode not in VISION_PROMPT_MODES:
errors.append(
f"Agent 节点 {node_id} 的视觉提示词模式无效:{vision_prompt_mode}"
)
if (
source.get("visionEnabled")
and vision_prompt_mode in {"fixed", "guided"}
and not str(source.get("visionPrompt") or "").strip()
):
errors.append(
f"Agent 节点 {node_id} 的固定或引导模式必须填写视觉提示词"
)
if runtime_mode == "realtime":
if data.get("contextPolicy", "inherit") != "inherit":
errors.append(
@@ -487,7 +510,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]:
errors.append(
f"Realtime Agent 节点 {node_id} 不支持节点级模型、语音或交互策略覆盖"
)
source = settings if data.get("inheritGlobalConfig", True) else data
if (
source.get("knowledgeBaseId")
and source.get("knowledgeMode") != "on_demand"
backend/services/pipecat/pipeline.py
+61 -25
View File
@@ -38,8 +38,9 @@ from services.vision import (
image_data_uri,
image_frame_from_jpeg,
image_jpeg_bytes,
resolve_vision_question,
vision_system_hint,
vision_tool_description,
vision_tool_schema_parts,
)
from services.workflow_engine import WorkflowEngine
@@ -349,7 +350,9 @@ async def run_pipeline(
def with_vision_hint(text: str) -> str:
hints = []
if vision_enabled and cfg.type != "workflow":
hints.append(vision_system_hint(cfg.vision_focus_prompt))
hints.append(
vision_system_hint(cfg.vision_prompt_mode, cfg.vision_prompt)
)
if cfg.knowledge_base_id:
hints.append(
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
@@ -450,7 +453,8 @@ async def run_pipeline(
workflow_vision_scope: dict[str, Any] = {
"enabled": False,
"vision_model_resource_id": None,
"vision_focus_prompt": "",
"vision_prompt_mode": "runtime",
"vision_prompt": "",
"llm_resource_id": None,
}
@@ -458,16 +462,15 @@ async def run_pipeline(
if cfg.type == "workflow":
return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope)
return vision_native_mode
vision_description, vision_properties, vision_required = vision_tool_schema_parts(
cfg.vision_prompt_mode,
cfg.vision_prompt,
)
vision_schema = FunctionSchema(
name=VISION_TOOL_NAME,
description=vision_tool_description(cfg.vision_focus_prompt),
properties={
"question": {
"type": "string",
"description": "用户关于当前视频画面的具体问题。",
}
},
required=["question"],
description=vision_description,
properties=vision_properties,
required=vision_required,
)
knowledge_schema = FunctionSchema(
name=KNOWLEDGE_TOOL_NAME,
@@ -498,7 +501,15 @@ async def run_pipeline(
await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"})
async def fetch_user_image(params: FunctionCallParams):
question = str(params.arguments.get("question") or "请描述当前画面。")
try:
question = resolve_vision_question(
cfg.vision_prompt_mode,
cfg.vision_prompt,
params.arguments,
)
except ValueError as exc:
await params.result_callback({"status": "error", "message": str(exc)})
return
user_id = vision_state.get("client_id")
if not user_id:
await params.result_callback(
@@ -576,14 +587,26 @@ async def run_pipeline(
result = config_with_main_llm_as_vision(cfg)
else:
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
result.vision_focus_prompt = str(
workflow_vision_scope.get("vision_focus_prompt") or ""
result.vision_prompt_mode = str(
workflow_vision_scope.get("vision_prompt_mode") or "runtime"
)
result.vision_prompt = str(
workflow_vision_scope.get("vision_prompt") or ""
).strip()
return result
if cfg.type == "workflow" and vision_enabled:
async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:
question = str(params.arguments.get("question") or "请描述当前画面。")
try:
vision_cfg = active_vision_config()
question = resolve_vision_question(
vision_cfg.vision_prompt_mode,
vision_cfg.vision_prompt,
params.arguments,
)
except ValueError as exc:
await params.result_callback({"status": "error", "message": str(exc)})
return
user_id = vision_state.get("client_id")
if not user_id:
await params.result_callback(
@@ -609,7 +632,15 @@ async def run_pipeline(
"status": "disabled",
"message": "当前 Agent 节点未开启视觉理解。",
}
question = str((args or {}).get("question") or "请描述当前画面。")
vision_cfg = active_vision_config()
try:
question = resolve_vision_question(
vision_cfg.vision_prompt_mode,
vision_cfg.vision_prompt,
args,
)
except ValueError as exc:
return {"status": "error", "message": str(exc)}
user_id = vision_state.get("client_id")
if not user_id:
return {
@@ -625,7 +656,7 @@ async def run_pipeline(
try:
frame = await vision_capture.request_image(llm, request)
observation = await analyze_image_with_vision_model(
active_vision_config(),
vision_cfg,
frame,
question,
)
@@ -654,14 +685,19 @@ async def run_pipeline(
active_vision_uses_main_llm,
)
workflow_vision_function = FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=vision_schema.description,
properties=vision_schema.properties,
required=vision_schema.required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
def workflow_vision_function(stage) -> FlowsFunctionSchema:
description, properties, required = vision_tool_schema_parts(
stage.vision_prompt_mode,
stage.vision_prompt,
)
return FlowsFunctionSchema(
name=VISION_TOOL_NAME,
description=description,
properties=properties,
required=required,
handler=flow_fetch_user_image,
cancel_on_interruption=True,
)
if vision_enabled and cfg.type != "workflow":
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
backend/services/vision.py
+85 -74
View File
@@ -5,8 +5,14 @@ from __future__ import annotations
import asyncio
import base64
from io import BytesIO
from typing import Any
from models import AssistantConfig, RuntimeModelResource
from models import (
VISION_PROMPT_MODES,
AssistantConfig,
RuntimeModelResource,
VisionPromptMode,
)
from openai import AsyncOpenAI
from PIL import Image
from pipecat.frames.frames import UserImageRawFrame
@@ -22,12 +28,6 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
"如果画面不足以判断，请明确说明不确定。"
)
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
"不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
"意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
)
def _require(value: str, label: str) -> str:
@@ -36,83 +36,96 @@ def _require(value: str, label: str) -> str:
raise ValueError(f"缺少模型资源配置: {label}")
def vision_system_hint(focus_prompt: str = "") -> str:
"""Describe when to use vision and keep a configured business focus visible."""
def normalize_vision_prompt_mode(value: object) -> VisionPromptMode:
"""Return a supported mode while keeping older saved configs compatible."""
focus = focus_prompt.strip()
if not focus:
return VISION_SYSTEM_HINT
mode = str(value or "runtime")
if mode == "fixed":
return "fixed"
if mode == "guided":
return "guided"
return "runtime"
def _compact_prompt(prompt: str, *, limit: int = 4000) -> str:
return " ".join(prompt.split())[:limit]
def vision_system_hint(
mode: VisionPromptMode = "runtime",
configured_prompt: str = "",
) -> str:
"""Tell the conversation model how the camera tool forms its question."""
prompt_mode = normalize_vision_prompt_mode(mode)
prompt = _compact_prompt(configured_prompt)
if prompt_mode == "fixed":
return (
f"{VISION_SYSTEM_HINT}\n"
f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。"
)
if prompt_mode == "guided":
return (
f"{VISION_SYSTEM_HINT}\n"
"调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导："
f"{prompt}"
)
return (
f"{VISION_SYSTEM_HINT}\n"
f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
"调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。"
)
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
"""Keep the safety baseline fixed while appending an editable business focus."""
def vision_tool_schema_parts(
mode: VisionPromptMode = "runtime",
configured_prompt: str = "",
) -> tuple[str, dict[str, Any], list[str]]:
"""Build the tool contract for one configured visual-question mode."""
focus = focus_prompt.strip()
if not focus:
return VISION_ANALYSIS_SYSTEM_PROMPT
return (
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
f"[业务视觉关注目标]\n{focus}\n"
"该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
)
def vision_tool_description(focus_prompt: str = "") -> str:
base = (
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
"人/物品/环境状态或需要视觉判断时调用。"
)
focus = " ".join(focus_prompt.split())
return f"{base} 当前业务重点：{focus[:600]}" if focus else base
async def generate_vision_focus_prompt(
resource: RuntimeModelResource,
source_prompt: str,
) -> str:
"""Generate one editable focus draft with an existing LLM resource."""
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
source = source_prompt.strip()
if not source:
raise ValueError("助手提示词不能为空")
values = resource.values or {}
secrets = resource.secrets or {}
extra_body = values.get("extraBody")
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
client = AsyncOpenAI(
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
)
try:
response = await client.chat.completions.create(
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
messages=[
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
{"role": "user", "content": source},
],
**extra,
prompt_mode = normalize_vision_prompt_mode(mode)
prompt = _compact_prompt(configured_prompt, limit=600)
if prompt_mode == "fixed":
return (
f"{base} 每次都检查同一个固定问题：{prompt}",
{},
[],
)
finally:
await client.close()
content = response.choices[0].message.content if response.choices else ""
result = (
content.strip()
if isinstance(content, str)
else str(content or "").strip()
question_description = "结合当前对话生成的、关于当前视频画面的具体问题。"
description = base
if prompt_mode == "guided":
description = f"{base} 生成问题时必须遵守固定引导：{prompt}"
question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}"
return (
description,
{"question": {"type": "string", "description": question_description}},
["question"],
)
if not result:
raise ValueError("模型没有生成有效的视觉关注目标")
if result.startswith("```") and result.endswith("```"):
result = result[3:-3].strip()
if result.startswith("text"):
result = result[4:].strip()
return result[:4000]
def resolve_vision_question(
mode: VisionPromptMode,
configured_prompt: str,
arguments: dict[str, Any] | None,
) -> str:
"""Resolve the exact question sent with the captured image."""
prompt_mode = normalize_vision_prompt_mode(mode)
if prompt_mode == "fixed":
question = configured_prompt.strip()
else:
question = str((arguments or {}).get("question") or "").strip()
if not question:
raise ValueError(
"固定视觉提示词为空"
if prompt_mode == "fixed"
else "视觉问题为空"
)
return question
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
@@ -173,9 +186,7 @@ async def analyze_image_with_vision_model(
messages=[
{
"role": "system",
"content": vision_analysis_system_prompt(
cfg.vision_focus_prompt
),
"content": VISION_ANALYSIS_SYSTEM_PROMPT,
},
{
"role": "user",
backend/services/workflow/agent.py
+3 -2
View File
@@ -49,7 +49,7 @@ class WorkflowAgentStage:
if stage.vision_enabled:
stage_prompt = (
f"{stage_prompt}\n\n[视觉能力]\n"
f"{vision_system_hint(stage.vision_focus_prompt)}"
f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}"
)
instruction = (
EDGE_TOOL_STAGE_INSTRUCTION
@@ -112,7 +112,8 @@ class WorkflowAgentStage:
{
"enabled": stage.vision_enabled,
"vision_model_resource_id": stage.vision_model_resource_id,
"vision_focus_prompt": stage.vision_focus_prompt,
"vision_prompt_mode": stage.vision_prompt_mode,
"vision_prompt": stage.vision_prompt,
"llm_resource_id": stage.llm_resource_id,
}
)
backend/services/workflow/realtime.py
+21 -14
View File
@@ -34,8 +34,9 @@ from services.vision import (
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_vision_resource,
resolve_vision_question,
vision_system_hint,
vision_tool_description,
vision_tool_schema_parts,
)
from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
@@ -414,7 +415,10 @@ class WorkflowRealtimeController:
f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
)
if stage.vision_enabled:
prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
prompt = (
f"{prompt}\n"
f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}"
)
return prompt
def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
@@ -573,10 +577,15 @@ class WorkflowRealtimeController:
{"status": "stale", "message": "当前 Agent 已经切换。"},
continue_response=False,
)
question = str(arguments.get("question") or "").strip()
if not question:
try:
question = resolve_vision_question(
stage.vision_prompt_mode,
stage.vision_prompt,
arguments,
)
except ValueError as exc:
return RealtimeToolResult(
{"status": "error", "message": "视觉问题为空"}
{"status": "error", "message": str(exc)}
)
if self._runtime.capture_image is None:
return RealtimeToolResult(
@@ -605,7 +614,6 @@ class WorkflowRealtimeController:
try:
if vision_cfg is not None:
vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
observation = await analyze_image_with_vision_model(
vision_cfg,
frame,
@@ -634,17 +642,16 @@ class WorkflowRealtimeController:
}
)
description, properties, required = vision_tool_schema_parts(
stage.vision_prompt_mode,
stage.vision_prompt,
)
return (
RealtimeTool(
name=VISION_TOOL_NAME,
description=vision_tool_description(stage.vision_focus_prompt),
properties={
"question": {
"type": "string",
"description": "需要根据当前画面判断的具体问题",
}
},
required=("question",),
description=description,
properties=properties,
required=tuple(required),
),
handler,
)
backend/services/workflow_engine.py
+4 -2
View File
@@ -23,7 +23,8 @@ class AgentStageConfig:
tts_resource_id: str
vision_enabled: bool
vision_model_resource_id: str | None
vision_focus_prompt: str
vision_prompt_mode: str
vision_prompt: str
tool_ids: tuple[str, ...]
state_variable_names: tuple[str, ...]
knowledge_base_id: str | None
@@ -152,7 +153,8 @@ class WorkflowEngine:
vision_model_resource_id=(
str(source.get("visionModelResourceId") or "") or None
),
vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(),
vision_prompt_mode=str(source.get("visionPromptMode") or "runtime"),
vision_prompt=str(source.get("visionPrompt") or "").strip(),
tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []),
# Variable write scope always belongs to the Agent node, even when
# its selected tool resources come from Workflow defaults.
backend/tests/test_brains.py
+9 -4
View File
@@ -3,7 +3,7 @@ from __future__ import annotations
import asyncio
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
from unittest.mock import AsyncMock, MagicMock, patch
from models import AssistantConfig, RuntimeTool
from pipecat.flows import FlowManager
@@ -1782,7 +1782,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
}
)
scopes = []
vision_function = object()
vision_function = MagicMock(side_effect=lambda _stage: object())
async def queue_frame(_frame):
pass
@@ -1800,13 +1800,18 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
await brain._apply_agent_stage("agent")
inherited_config = brain._agent_config("agent")
self.assertIn(vision_function, inherited_config["functions"])
self.assertEqual(len(inherited_config["functions"]), 1)
vision_function.assert_called_once()
selected_stage = vision_function.call_args.args[0]
self.assertEqual(selected_stage.vision_prompt_mode, "runtime")
self.assertIn("fetch_user_image", inherited_config["role_message"])
self.assertEqual(
scopes[-1],
{
"enabled": True,
"vision_model_resource_id": "vision_global",
"vision_prompt_mode": "runtime",
"vision_prompt": "",
"llm_resource_id": "llm_global",
},
)
@@ -1821,7 +1826,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
)
await brain._apply_agent_stage("agent")
custom_config = brain._agent_config("agent")
self.assertNotIn(vision_function, custom_config["functions"])
self.assertEqual(custom_config["functions"], [])
self.assertNotIn("fetch_user_image", custom_config["role_message"])
self.assertFalse(scopes[-1]["enabled"])
backend/tests/test_vision_focus.py
-126
View File
@@ -1,126 +0,0 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from models import RuntimeModelResource
from services.vision import (
VISION_ANALYSIS_SYSTEM_PROMPT,
VISION_SYSTEM_HINT,
generate_vision_focus_prompt,
vision_analysis_system_prompt,
vision_system_hint,
vision_tool_description,
)
from services.workflow_engine import WorkflowEngine
class VisionFocusPromptTest(unittest.TestCase):
def test_focus_extends_but_does_not_replace_safety_prompts(self):
focus = "优先识别车辆数量和人员状态，只报告可确认的信息。"
system_hint = vision_system_hint(focus)
analysis_prompt = vision_analysis_system_prompt(focus)
self.assertIn(VISION_SYSTEM_HINT, system_hint)
self.assertIn(focus, system_hint)
self.assertIn(VISION_ANALYSIS_SYSTEM_PROMPT, analysis_prompt)
self.assertIn(focus, analysis_prompt)
self.assertIn("优先回答当前用户问题", analysis_prompt)
def test_empty_focus_preserves_previous_runtime_prompts(self):
self.assertEqual(vision_system_hint(), VISION_SYSTEM_HINT)
self.assertEqual(
vision_analysis_system_prompt(),
VISION_ANALYSIS_SYSTEM_PROMPT,
)
def test_tool_description_compacts_and_limits_focus(self):
description = vision_tool_description("车辆\n 数量 " * 300)
self.assertIn("当前业务重点", description)
self.assertNotIn("\n", description)
self.assertLess(len(description), 700)
def test_workflow_resolves_global_and_node_focus(self):
graph = {
"specVersion": 3,
"settings": {
"visionEnabled": True,
"visionFocusPrompt": "全局关注目标",
},
"nodes": [
{
"id": "global-agent",
"type": "agent",
"data": {"name": "全局 Agent", "prompt": "处理任务"},
},
{
"id": "custom-agent",
"type": "agent",
"data": {
"name": "独立 Agent",
"prompt": "检查车辆",
"inheritGlobalConfig": False,
"visionEnabled": True,
"visionFocusPrompt": "节点关注目标",
},
},
],
"edges": [],
}
engine = WorkflowEngine(graph)
self.assertEqual(
engine.agent_stage_config("global-agent").vision_focus_prompt,
"全局关注目标",
)
self.assertEqual(
engine.agent_stage_config("custom-agent").vision_focus_prompt,
"节点关注目标",
)
class VisionFocusGenerationTest(unittest.IsolatedAsyncioTestCase):
async def test_generates_editable_focus_with_selected_resource(self):
resource = RuntimeModelResource(
id="llm_1",
name="生成模型",
capability="LLM",
interface_type="openai-llm",
values={
"apiUrl": "https://llm.example/v1",
"modelId": "focus-model",
},
secrets={"apiKey": "secret"},
)
client = MagicMock()
client.chat.completions.create = AsyncMock(
return_value=SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(
content="优先识别车辆数量、损伤位置和人员状态。"
)
)
]
)
)
client.close = AsyncMock()
with patch("services.vision.AsyncOpenAI", return_value=client):
result = await generate_vision_focus_prompt(
resource,
"你是交通事故处理助手，需要了解车辆和人员情况。",
)
self.assertEqual(result, "优先识别车辆数量、损伤位置和人员状态。")
request = client.chat.completions.create.await_args.kwargs
self.assertEqual(request["model"], "focus-model")
self.assertIn("交通事故处理助手", request["messages"][1]["content"])
client.close.assert_awaited_once()
if __name__ == "__main__":
unittest.main()
backend/tests/test_vision_prompt_modes.py
+227
View File
@@ -0,0 +1,227 @@
from __future__ import annotations
import unittest
from types import SimpleNamespace
from models import AssistantConfig
from services.brains.base import RealtimeBrainRuntime
from services.node_specs import normalize_graph, validate_graph
from services.vision import (
VISION_SYSTEM_HINT,
resolve_vision_question,
vision_system_hint,
vision_tool_schema_parts,
)
from services.workflow_engine import WorkflowEngine
from services.workflow.realtime import WorkflowRealtimeController
class VisionPromptModeTest(unittest.TestCase):
def test_fixed_mode_has_no_runtime_question_parameter(self):
description, properties, required = vision_tool_schema_parts(
"fixed",
"识别车辆数量和人员状态",
)
self.assertIn("识别车辆数量和人员状态", description)
self.assertEqual(properties, {})
self.assertEqual(required, [])
self.assertEqual(
resolve_vision_question("fixed", " 识别车辆数量 ", {}),
"识别车辆数量",
)
def test_runtime_mode_requires_model_generated_question(self):
description, properties, required = vision_tool_schema_parts("runtime")
self.assertNotIn("固定引导", description)
self.assertIn("question", properties)
self.assertEqual(required, ["question"])
self.assertEqual(
resolve_vision_question(
"runtime",
"不会使用这个配置",
{"question": "用户手里拿着什么？"},
),
"用户手里拿着什么？",
)
def test_guided_mode_exposes_guide_but_uses_runtime_question(self):
guide = "只询问画面中可直接确认的车辆损伤"
description, properties, required = vision_tool_schema_parts(
"guided",
guide,
)
self.assertIn(guide, description)
self.assertIn(guide, properties["question"]["description"])
self.assertEqual(required, ["question"])
self.assertEqual(
resolve_vision_question(
"guided",
guide,
{"question": "右侧车门是否有明显凹陷？"},
),
"右侧车门是否有明显凹陷？",
)
def test_mode_specific_system_hints(self):
self.assertIn(VISION_SYSTEM_HINT, vision_system_hint("runtime"))
self.assertIn("不需要生成", vision_system_hint("fixed", "固定问题"))
self.assertIn("固定引导", vision_system_hint("guided", "业务引导"))
def test_missing_required_config_or_question_is_rejected(self):
with self.assertRaisesRegex(ValueError, "固定视觉提示词为空"):
resolve_vision_question("fixed", "", {})
with self.assertRaisesRegex(ValueError, "视觉问题为空"):
resolve_vision_question("runtime", "", {})
class WorkflowVisionPromptModeTest(unittest.TestCase):
def test_workflow_resolves_global_and_node_modes(self):
graph = {
"specVersion": 3,
"settings": {
"visionEnabled": True,
"visionPromptMode": "fixed",
"visionPrompt": "全局固定问题",
},
"nodes": [
{
"id": "global-agent",
"type": "agent",
"data": {"name": "全局 Agent", "prompt": "处理任务"},
},
{
"id": "custom-agent",
"type": "agent",
"data": {
"name": "独立 Agent",
"prompt": "检查车辆",
"inheritGlobalConfig": False,
"visionEnabled": True,
"visionPromptMode": "guided",
"visionPrompt": "节点引导",
},
},
],
"edges": [],
}
engine = WorkflowEngine(graph)
global_stage = engine.agent_stage_config("global-agent")
self.assertEqual(global_stage.vision_prompt_mode, "fixed")
self.assertEqual(global_stage.vision_prompt, "全局固定问题")
custom_stage = engine.agent_stage_config("custom-agent")
self.assertEqual(custom_stage.vision_prompt_mode, "guided")
self.assertEqual(custom_stage.vision_prompt, "节点引导")
def test_legacy_focus_is_migrated_to_guided_mode(self):
normalized = normalize_graph(
{
"specVersion": 3,
"settings": {"visionFocusPrompt": "旧全局关注目标"},
"nodes": [],
"edges": [],
}
)
self.assertEqual(normalized["settings"]["visionPromptMode"], "guided")
self.assertEqual(normalized["settings"]["visionPrompt"], "旧全局关注目标")
self.assertNotIn("visionFocusPrompt", normalized["settings"])
def test_fixed_or_guided_mode_requires_prompt_when_vision_is_enabled(self):
graph = {
"specVersion": 3,
"settings": {
"visionEnabled": True,
"visionPromptMode": "fixed",
"visionPrompt": "",
},
"nodes": [
{
"id": "agent",
"type": "agent",
"data": {"name": "Agent", "prompt": "处理任务"},
}
],
"edges": [],
}
self.assertTrue(
any("必须填写视觉提示词" in error for error in validate_graph(graph))
)
def test_invalid_mode_is_reported_instead_of_silently_rewritten(self):
graph = {
"specVersion": 3,
"settings": {"visionPromptMode": "unexpected"},
"nodes": [
{
"id": "agent",
"type": "agent",
"data": {"name": "Agent", "prompt": "处理任务"},
}
],
"edges": [],
}
self.assertTrue(
any("视觉提示词模式无效" in error for error in validate_graph(graph))
)
def test_realtime_tool_contract_changes_with_mode(self):
graph = {
"specVersion": 3,
"settings": {},
"nodes": [
{
"id": "fixed-agent",
"type": "agent",
"data": {
"name": "固定",
"prompt": "处理任务",
"inheritGlobalConfig": False,
"visionEnabled": True,
"visionPromptMode": "fixed",
"visionPrompt": "检查安全帽",
},
},
{
"id": "guided-agent",
"type": "agent",
"data": {
"name": "引导",
"prompt": "处理任务",
"inheritGlobalConfig": False,
"visionEnabled": True,
"visionPromptMode": "guided",
"visionPrompt": "只询问安全设备",
},
},
],
"edges": [],
}
engine = WorkflowEngine(graph)
controller = WorkflowRealtimeController(
cfg=AssistantConfig(type="workflow"),
engine=engine,
store=SimpleNamespace(),
runtime=RealtimeBrainRuntime(
realtime=SimpleNamespace(),
queue_frame=lambda _frame: None,
call_end=SimpleNamespace(ending=False),
),
)
fixed_tool, _fixed_handler = controller._vision_tool("fixed-agent", 0)
guided_tool, _guided_handler = controller._vision_tool("guided-agent", 0)
self.assertEqual(fixed_tool.properties, {})
self.assertEqual(fixed_tool.required, ())
self.assertIn("只询问安全设备", guided_tool.description)
self.assertEqual(guided_tool.required, ("question",))
if __name__ == "__main__":
unittest.main()
frontend/src/components/assistant-editor/prompt-editor.tsx
+7 -5
View File
@@ -648,17 +648,19 @@ export function PromptEditor({
hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。"
enabled={form.visionEnabled}
modelResourceId={form.visionModelResourceId}
focusPrompt={form.visionFocusPrompt}
sourcePrompt={form.prompt}
generatorModelResourceId={form.model}
promptMode={form.visionPromptMode}
prompt={form.visionPrompt}
mainModelResourceId={form.model}
modelOptions={visionModelOptions}
onEnabledChange={handlePromptVisionEnabledChange}
onModelResourceIdChange={(value) =>
updateForm("visionModelResourceId", value)
}
onFocusPromptChange={(value) =>
updateForm("visionFocusPrompt", value)
onPromptModeChange={(value) =>
updateForm("visionPromptMode", value)
}
onPromptChange={(value) =>
updateForm("visionPrompt", value)
}
/>
)}
frontend/src/components/assistant-editor/types.ts
+3 -1
View File
@@ -4,6 +4,7 @@ import type {
KnowledgeRetrievalConfig,
StartupConfig,
TurnConfig,
VisionPromptMode,
} from "@/lib/api";
export type RuntimeMode = "pipeline" | "realtime";
@@ -26,6 +27,7 @@ export type AssistantForm = {
startup: StartupConfig;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
visionPromptMode: VisionPromptMode;
visionPrompt: string;
toolIds: string[];
};
frontend/src/components/editor/vision-config-section.tsx
+103 -81
View File
@@ -1,53 +1,71 @@
"use client";
import { useState } from "react";
import { Camera, Loader2, Sparkles } from "lucide-react";
import { Camera, Check } from "lucide-react";
import {
ResourceSelectField,
ToggleRow,
} from "@/components/assistant-editor/editor-controls";
import { SectionCard } from "@/components/editor/section-card";
import { Button } from "@/components/ui/button";
import { Textarea } from "@/components/ui/textarea";
import { assistantsApi } from "@/lib/api";
import type { VisionPromptMode } from "@/lib/api";
type VisionModelOption = {
value: string;
label: string;
};
const promptModes: Array<{
value: VisionPromptMode;
title: string;
description: string;
}> = [
{
value: "fixed",
title: "固定提示词",
description: "每次获取画面都发送同一个问题。",
},
{
value: "runtime",
title: "运行时自动生成",
description: "模型结合当前对话，临时生成看图问题。",
},
{
value: "guided",
title: "固定提示词引导",
description: "模型动态生成问题，同时遵守固定引导。",
},
];
export function VisionConfigSection({
description,
hint,
enabled,
modelResourceId,
focusPrompt,
sourcePrompt,
generatorModelResourceId,
promptMode,
prompt,
mainModelResourceId,
mainModelSupportsImageInput,
modelOptions,
onEnabledChange,
onModelResourceIdChange,
onFocusPromptChange,
onPromptModeChange,
onPromptChange,
}: {
description: string;
hint: string;
enabled: boolean;
modelResourceId: string;
focusPrompt: string;
sourcePrompt: string;
generatorModelResourceId: string;
promptMode: VisionPromptMode;
prompt: string;
mainModelResourceId: string;
mainModelSupportsImageInput?: boolean;
modelOptions: VisionModelOption[];
onEnabledChange: (enabled: boolean) => void;
onModelResourceIdChange: (modelResourceId: string) => void;
onFocusPromptChange: (focusPrompt: string) => void;
onPromptModeChange: (promptMode: VisionPromptMode) => void;
onPromptChange: (prompt: string) => void;
}) {
const [generating, setGenerating] = useState(false);
const [generationError, setGenerationError] = useState("");
const mainModelSupportsVision =
mainModelSupportsImageInput ??
modelOptions.some((option) => option.value === mainModelResourceId);
@@ -55,25 +73,6 @@ export function VisionConfigSection({
(option) => option.value !== mainModelResourceId,
);
async function generateFocusPrompt() {
if (!sourcePrompt.trim() || !generatorModelResourceId || generating) return;
setGenerating(true);
setGenerationError("");
try {
const result = await assistantsApi.generateVisionFocus({
sourcePrompt,
modelResourceId: generatorModelResourceId,
});
onFocusPromptChange(result.focusPrompt);
} catch (error) {
setGenerationError(
error instanceof Error ? error.message : "生成视觉关注目标失败",
);
} finally {
setGenerating(false);
}
}
return (
<SectionCard
icon={<Camera size={15} />}
@@ -100,56 +99,79 @@ export function VisionConfigSection({
当前模型未标记支持图片输入，请选择独立视觉模型。
</p>
)}
<div className="space-y-1.5">
<div className="flex items-center justify-between gap-3">
<label className="text-sm font-medium text-foreground">
视觉关注目标
</label>
<Button
type="button"
variant="ghost"
size="sm"
className="h-7 gap-1.5 rounded-full px-2.5 text-xs"
disabled={
generating || !sourcePrompt.trim() || !generatorModelResourceId
}
onClick={() => void generateFocusPrompt()}
title={
!sourcePrompt.trim()
? "请先填写助手提示词"
: !generatorModelResourceId
? "请先选择大语言模型"
: "根据当前助手提示词生成可编辑草稿"
}
>
{generating ? (
<Loader2 size={12} className="animate-spin" />
) : (
<Sparkles size={12} />
)}
{generating ? "生成中" : "根据提示词生成"}
</Button>
<div className="space-y-2">
<label className="text-sm font-medium text-foreground">
看图问题生成方式
</label>
<div role="radiogroup" aria-label="看图问题生成方式" className="space-y-2">
{promptModes.map((option) => {
const selected = promptMode === option.value;
return (
<button
key={option.value}
type="button"
role="radio"
aria-checked={selected}
onClick={() => onPromptModeChange(option.value)}
className={[
"flex w-full items-start gap-3 rounded-xl border px-3.5 py-3 text-left transition-colors",
selected
? "border-primary bg-surface-strong/70 ring-1 ring-primary"
: "border-hairline bg-background hover:border-hairline-strong",
].join(" ")}
>
<span
className={[
"mt-0.5 flex h-5 w-5 shrink-0 items-center justify-center rounded-full border",
selected
? "border-primary bg-primary text-primary-foreground"
: "border-hairline-strong text-transparent",
].join(" ")}
>
<Check size={12} />
</span>
<span className="min-w-0">
<span className="block text-sm font-medium text-foreground">
{option.title}
</span>
<span className="mt-0.5 block text-xs leading-5 text-muted-foreground">
{option.description}
</span>
</span>
</button>
);
})}
</div>
<Textarea
value={focusPrompt}
maxLength={4000}
rows={5}
onChange={(event) => {
onFocusPromptChange(event.target.value);
if (generationError) setGenerationError("");
}}
placeholder="例如：优先识别车辆数量、碰撞位置、明显损伤和人员状态；只报告画面中可直接确认的信息。"
className="field-sizing-fixed min-h-28 resize-y border-hairline-strong bg-background text-sm leading-6 text-foreground placeholder:text-muted-soft"
/>
<p className="text-[11px] leading-5 text-muted-soft">
可选。系统仍保留通用视觉安全规则；这里用于让每次看图稳定聚焦业务目标。
</p>
{generationError && (
<p className="text-xs leading-5 text-destructive">
{generationError}
</p>
)}
</div>
{promptMode !== "runtime" && (
<div className="space-y-1.5">
<label className="text-sm font-medium text-foreground">
{promptMode === "fixed" ? "固定视觉提示词" : "运行时提示词引导"}
</label>
<Textarea
value={prompt}
maxLength={4000}
rows={5}
onChange={(event) => onPromptChange(event.target.value)}
placeholder={
promptMode === "fixed"
? "例如：请识别画面中的车辆数量、碰撞位置、明显损伤和人员状态。"
: "例如：问题应聚焦车辆数量、碰撞位置和人员状态；只询问画面中可直接判断的信息。"
}
className="field-sizing-fixed min-h-28 resize-y border-hairline-strong bg-background text-sm leading-6 text-foreground placeholder:text-muted-soft"
/>
<p className="text-[11px] leading-5 text-muted-soft">
{promptMode === "fixed"
? "工具调用时不会让模型改写这段内容。"
: "模型会结合当前对话生成具体问题，并受到这段内容约束。"}
</p>
{!prompt.trim() && (
<p className="text-xs leading-5 text-destructive">
当前模式需要填写提示词后才能保存。
</p>
)}
</div>
)}
</>
)}
</SectionCard>
frontend/src/components/pages/AssistantPage.tsx
+10 -5
View File
@@ -180,7 +180,8 @@ function blankPromptForm(name: string): AssistantForm {
},
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
toolIds: [],
};
}
@@ -436,7 +437,8 @@ export function AssistantPage(props: AssistantPageProps) {
},
visionEnabled: a.visionEnabled,
visionModelResourceId: a.visionModelResourceId ?? "",
visionFocusPrompt: a.visionFocusPrompt ?? "",
visionPromptMode: a.visionPromptMode ?? "runtime",
visionPrompt: a.visionPrompt ?? "",
toolIds: a.toolIds ?? [],
};
setForm(next);
@@ -487,7 +489,8 @@ export function AssistantPage(props: AssistantPageProps) {
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
@@ -552,7 +555,8 @@ export function AssistantPage(props: AssistantPageProps) {
startup: form.startup,
visionEnabled: form.visionEnabled,
visionModelResourceId: form.visionModelResourceId || null,
visionFocusPrompt: form.visionFocusPrompt,
visionPromptMode: form.visionPromptMode,
visionPrompt: form.visionPrompt,
modelResourceIds: {
...(form.model ? { LLM: form.model } : {}),
...(form.asr ? { ASR: form.asr } : {}),
@@ -680,7 +684,8 @@ export function AssistantPage(props: AssistantPageProps) {
graph.settings?.visionModelResourceId ??
assistant.visionModelResourceId ??
"",
visionFocusPrompt: graph.settings?.visionFocusPrompt ?? "",
visionPromptMode: graph.settings?.visionPromptMode ?? "runtime",
visionPrompt: graph.settings?.visionPrompt ?? "",
toolIds: graph.settings?.toolIds ?? [],
knowledgeBaseId:
graph.settings?.knowledgeBaseId || assistant.knowledgeBaseId || "",
frontend/src/components/pages/CreateAssistantPage.tsx
+2 -1
View File
@@ -108,7 +108,8 @@ function baseUpsert(over: Partial<AssistantUpsert>): AssistantUpsert {
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
frontend/src/components/pages/TemplateLibraryPage.tsx
+2 -1
View File
@@ -96,7 +96,8 @@ function baseUpsertFromTemplate(
},
visionEnabled: false,
visionModelResourceId: null,
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
modelResourceIds: {},
knowledgeBaseId: null,
knowledgeRetrievalConfig: defaultKnowledgeRetrievalConfig(),
frontend/src/components/workflow/WorkflowCanvas.tsx
+2 -1
View File
@@ -129,7 +129,8 @@ function fromFlow(nodes: Node[], edges: Edge[]): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
frontend/src/components/workflow/graph-settings.ts
+4 -2
View File
@@ -15,7 +15,8 @@ export function settingsFromWorkflowGraph(
tts: graph.settings.defaultTtsResourceId,
visionEnabled: graph.settings.visionEnabled,
visionModelResourceId: graph.settings.visionModelResourceId,
visionFocusPrompt: graph.settings.visionFocusPrompt,
visionPromptMode: graph.settings.visionPromptMode,
visionPrompt: graph.settings.visionPrompt,
toolIds: graph.settings.toolIds,
knowledgeBaseId: graph.settings.knowledgeBaseId,
knowledgeRetrievalConfig: {
@@ -45,7 +46,8 @@ export function workflowGraphWithSettings(
defaultTtsResourceId: settings.tts ?? "",
visionEnabled: settings.visionEnabled,
visionModelResourceId: settings.visionModelResourceId,
visionFocusPrompt: settings.visionFocusPrompt,
visionPromptMode: settings.visionPromptMode,
visionPrompt: settings.visionPrompt,
toolIds: settings.toolIds,
knowledgeBaseId: settings.knowledgeBaseId,
knowledgeMode: settings.knowledgeRetrievalConfig.mode,
frontend/src/components/workflow/panels/AgentNodePanel.tsx
+11 -11
View File
@@ -78,8 +78,10 @@ export function AgentNodePanel({
visionModelResourceId:
(draft.visionModelResourceId as string) ||
workflowSettings.visionModelResourceId,
visionFocusPrompt:
draft.visionFocusPrompt ?? workflowSettings.visionFocusPrompt,
visionPromptMode:
draft.visionPromptMode ?? workflowSettings.visionPromptMode,
visionPrompt:
draft.visionPrompt ?? workflowSettings.visionPrompt,
...(!isRealtime
? {
llmResourceId:
@@ -308,13 +310,8 @@ export function AgentNodePanel({
}
enabled={Boolean(draft.visionEnabled)}
modelResourceId={draft.visionModelResourceId ?? ""}
focusPrompt={draft.visionFocusPrompt ?? ""}
sourcePrompt={draft.prompt ?? ""}
generatorModelResourceId={
isRealtime
? draft.visionModelResourceId ?? ""
: (draft.llmResourceId as string) || ""
}
promptMode={draft.visionPromptMode ?? "runtime"}
prompt={draft.visionPrompt ?? ""}
mainModelResourceId={
isRealtime
? workflowSettings.realtime ?? ""
@@ -339,8 +336,11 @@ export function AgentNodePanel({
onModelResourceIdChange={(visionModelResourceId) =>
set("visionModelResourceId", visionModelResourceId)
}
onFocusPromptChange={(visionFocusPrompt) =>
set("visionFocusPrompt", visionFocusPrompt)
onPromptModeChange={(visionPromptMode) =>
set("visionPromptMode", visionPromptMode)
}
onPromptChange={(visionPrompt) =>
set("visionPrompt", visionPrompt)
}
/>
frontend/src/components/workflow/panels/GlobalSettingsPanel.tsx
+7 -9
View File
@@ -184,13 +184,8 @@ export function GlobalSettingsPanel({
}
enabled={settings.visionEnabled}
modelResourceId={settings.visionModelResourceId}
focusPrompt={settings.visionFocusPrompt}
sourcePrompt={settings.globalPrompt}
generatorModelResourceId={
settings.runtimeMode === "realtime"
? settings.visionModelResourceId
: settings.llm ?? ""
}
promptMode={settings.visionPromptMode}
prompt={settings.visionPrompt}
mainModelResourceId={
settings.runtimeMode === "realtime"
? settings.realtime ?? ""
@@ -216,8 +211,11 @@ export function GlobalSettingsPanel({
onModelResourceIdChange={(visionModelResourceId) =>
onSettingsChange({ ...settings, visionModelResourceId })
}
onFocusPromptChange={(visionFocusPrompt) =>
onSettingsChange({ ...settings, visionFocusPrompt })
onPromptModeChange={(visionPromptMode) =>
onSettingsChange({ ...settings, visionPromptMode })
}
onPromptChange={(visionPrompt) =>
onSettingsChange({ ...settings, visionPrompt })
}
/>
frontend/src/components/workflow/specs.ts
+7 -4
View File
@@ -3,7 +3,7 @@
import * as LucideIcons from "lucide-react";
import { Circle, type LucideIcon } from "lucide-react";
import type { NodeSpecDto, TurnConfig } from "@/lib/api";
import type { NodeSpecDto, TurnConfig, VisionPromptMode } from "@/lib/api";
import { defaultTurnConfig } from "@/lib/turn-config";
export type WorkflowNodeType =
@@ -54,7 +54,8 @@ export type WorkflowNodeData = {
ttsResourceId?: string;
visionEnabled?: boolean;
visionModelResourceId?: string;
visionFocusPrompt?: string;
visionPromptMode?: VisionPromptMode;
visionPrompt?: string;
enableInterrupt?: boolean;
turnConfig?: TurnConfig;
toolId?: string;
@@ -214,7 +215,8 @@ export type WorkflowGraph = {
defaultTtsResourceId: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
visionPromptMode: VisionPromptMode;
visionPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeMode: "automatic" | "on_demand";
@@ -252,7 +254,8 @@ export function defaultGraph(): WorkflowGraph {
defaultTtsResourceId: "",
visionEnabled: false,
visionModelResourceId: "",
visionFocusPrompt: "",
visionPromptMode: "runtime",
visionPrompt: "",
toolIds: [],
knowledgeBaseId: "",
knowledgeMode: "automatic",
frontend/src/components/workflow/types.ts
+3 -1
View File
@@ -5,6 +5,7 @@ import type {
SystemToolKind,
Tool,
TurnConfig,
VisionPromptMode,
} from "@/lib/api";
import type { WorkflowGraph } from "./specs";
@@ -18,7 +19,8 @@ export type WorkflowSettings = {
tts?: string;
visionEnabled: boolean;
visionModelResourceId: string;
visionFocusPrompt: string;
visionPromptMode: VisionPromptMode;
visionPrompt: string;
toolIds: string[];
knowledgeBaseId: string;
knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
frontend/src/lib/api.ts
+4 -9
View File
@@ -315,6 +315,8 @@ export type AnalysisConfig = {
fields: AnalysisField[];
};
export type VisionPromptMode = "fixed" | "runtime" | "guided";
/** 后端 AssistantOut(宽表 STI:瘦字段平铺,workflow 用 graph)。apiKey 读时打码 */
export type Assistant = {
id: string;
@@ -327,7 +329,8 @@ export type Assistant = {
startup: StartupConfig;
visionEnabled: boolean;
visionModelResourceId: string | null;
visionFocusPrompt: string;
visionPromptMode: VisionPromptMode;
visionPrompt: string;
modelResourceIds: Partial<Record<ModelType, string>>;
knowledgeBaseId: string | null;
knowledgeRetrievalConfig: KnowledgeRetrievalConfig;
@@ -372,14 +375,6 @@ export const assistantsApi = {
// 服务端整行复制(含真 key,密钥不经浏览器)
duplicate: (id: string) =>
request<Assistant>(`/api/assistants/${id}/duplicate`, { method: "POST" }),
generateVisionFocus: (body: {
sourcePrompt: string;
modelResourceId: string;
}) =>
request<{ focusPrompt: string }>("/api/assistants/vision-focus/generate", {
method: "POST",
body: JSON.stringify(body),
}),
remove: (id: string) =>
request<{ ok: boolean }>(`/api/assistants/${id}`, { method: "DELETE" }),
};