diff --git a/backend/db/models.py b/backend/db/models.py index 2e6baa8..acba2cc 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -160,7 +160,8 @@ class Assistant(Base): ForeignKey("model_resources.id", ondelete="SET NULL"), nullable=True, ) - vision_focus_prompt: Mapped[str] = mapped_column(Text, default="") + vision_prompt_mode: Mapped[str] = mapped_column(String(16), default="runtime") + vision_prompt: Mapped[str] = mapped_column(Text, default="") # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底 knowledge_base_id: Mapped[str | None] = mapped_column( diff --git a/backend/migrations/versions/20260812_0020_add_vision_prompt_mode.py b/backend/migrations/versions/20260812_0020_add_vision_prompt_mode.py new file mode 100644 index 0000000..801c15a --- /dev/null +++ b/backend/migrations/versions/20260812_0020_add_vision_prompt_mode.py @@ -0,0 +1,47 @@ +"""replace vision focus with explicit prompt modes + +Revision ID: 20260812_0020 +Revises: 20260812_0019 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260812_0020" +down_revision: str | Sequence[str] | None = "20260812_0019" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.alter_column( + "assistants", + "vision_focus_prompt", + new_column_name="vision_prompt", + ) + op.add_column( + "assistants", + sa.Column( + "vision_prompt_mode", + sa.String(length=16), + server_default="runtime", + nullable=False, + ), + ) + # The previous single-field implementation behaved most like guided mode. + op.execute( + "UPDATE assistants SET vision_prompt_mode = 'guided' " + "WHERE TRIM(vision_prompt) <> ''" + ) + + +def downgrade() -> None: + op.drop_column("assistants", "vision_prompt_mode") + op.alter_column( + "assistants", + "vision_prompt", + new_column_name="vision_focus_prompt", + ) diff --git a/backend/models.py b/backend/models.py index f47a36e..1fb0bee 100644 --- a/backend/models.py +++ b/backend/models.py @@ -13,6 +13,8 @@ from typing import Literal from pydantic import BaseModel, Field RuntimeMode = Literal["pipeline", "realtime"] +VisionPromptMode = Literal["fixed", "runtime", "guided"] +VISION_PROMPT_MODES = frozenset({"fixed", "runtime", "guided"}) class RuntimeMcpServer(BaseModel): @@ -92,7 +94,8 @@ class AssistantConfig(BaseModel): llm_support_image_input: bool = False vision_enabled: bool = False vision_model_resource_id: str | None = None - vision_focus_prompt: str = "" + vision_prompt_mode: VisionPromptMode = "runtime" + vision_prompt: str = "" vision_model: str = "" vision_llm_interface_type: str = "openai-llm" vision_llm_values: dict = {} diff --git a/backend/routes/assistants.py b/backend/routes/assistants.py index 2e0ca67..775f623 100644 --- a/backend/routes/assistants.py +++ b/backend/routes/assistants.py @@ -2,7 +2,7 @@ import uuid -from models import RuntimeModelResource, RuntimeTool +from models import VISION_PROMPT_MODES, RuntimeTool from db.models import ( Assistant, AssistantModelBinding, @@ -16,15 +16,12 @@ from fastapi import APIRouter, Depends, HTTPException from schemas import ( AssistantOut, AssistantUpsert, - VisionFocusPromptGenerateRequest, - VisionFocusPromptGenerateResult, ) from services.auth import require_admin from services.masking import mask, resolve_incoming_key from services.node_specs import graph_references, normalize_graph, validate_graph from services.system_tools import system_tool_kind from services.tool_policy import policy_for_tool -from services.vision import generate_vision_focus_prompt from services.workflow_engine import WorkflowEngine from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -79,7 +76,8 @@ def _validate_workflow(body: AssistantUpsert) -> None: # capability summary used by clients to decide whether to open the camera. body.vision_enabled = WorkflowEngine(body.graph).uses_vision() body.vision_model_resource_id = None - body.vision_focus_prompt = "" + body.vision_prompt_mode = "runtime" + body.vision_prompt = "" refs = graph_references(body.graph) body.tool_ids = list(dict.fromkeys([*body.tool_ids, *sorted(refs["tools"])])) @@ -271,6 +269,13 @@ async def _validate_vision_model( if body.type == "workflow": return if body.vision_enabled: + if body.vision_prompt_mode not in VISION_PROMPT_MODES: + raise HTTPException(400, "视觉提示词模式无效") + if ( + body.vision_prompt_mode in {"fixed", "guided"} + and not body.vision_prompt.strip() + ): + raise HTTPException(400, "固定或引导模式必须填写视觉提示词") if body.vision_model_resource_id: resource = await session.get(ModelResource, body.vision_model_resource_id) else: @@ -396,7 +401,8 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut: startup=assistant.startup or {}, vision_enabled=assistant.vision_enabled, vision_model_resource_id=assistant.vision_model_resource_id, - vision_focus_prompt=assistant.vision_focus_prompt or "", + vision_prompt_mode=assistant.vision_prompt_mode or "runtime", + vision_prompt=assistant.vision_prompt or "", model_resource_ids=await _resource_ids(session, assistant.id), knowledge_base_id=assistant.knowledge_base_id, knowledge_retrieval_config=assistant.knowledge_retrieval_config or {}, @@ -448,43 +454,6 @@ async def create_assistant( return await _to_out(session, assistant) -@router.post( - "/vision-focus/generate", - response_model=VisionFocusPromptGenerateResult, -) -async def generate_assistant_vision_focus( - body: VisionFocusPromptGenerateRequest, - session: AsyncSession = Depends(get_session), -): - """Generate an editable visual focus draft from an assistant prompt.""" - - resource = await session.get(ModelResource, body.model_resource_id) - if ( - resource is None - or resource.capability != "LLM" - or not resource.enabled - ): - raise HTTPException(400, "请选择可用的大语言模型生成视觉关注目标") - try: - focus_prompt = await generate_vision_focus_prompt( - RuntimeModelResource( - id=resource.id, - name=resource.name, - capability=resource.capability, - interface_type=resource.interface_type, - values=resource.values or {}, - secrets=resource.secrets or {}, - support_image_input=bool(resource.support_image_input), - ), - body.source_prompt, - ) - except ValueError as exc: - raise HTTPException(400, str(exc)) from exc - except Exception as exc: - raise HTTPException(502, "生成视觉关注目标失败，请稍后重试") from exc - return VisionFocusPromptGenerateResult(focus_prompt=focus_prompt) - - @router.get("/{assistant_id}", response_model=AssistantOut) async def get_assistant( assistant_id: str, session: AsyncSession = Depends(get_session) @@ -513,7 +482,8 @@ async def duplicate_assistant( startup=dict(source.startup or {}), vision_enabled=source.vision_enabled, vision_model_resource_id=source.vision_model_resource_id, - vision_focus_prompt=source.vision_focus_prompt or "", + vision_prompt_mode=source.vision_prompt_mode or "runtime", + vision_prompt=source.vision_prompt or "", knowledge_base_id=source.knowledge_base_id, knowledge_retrieval_config=dict(source.knowledge_retrieval_config or {}), prompt=source.prompt, diff --git a/backend/schemas.py b/backend/schemas.py index de8077b..caee996 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida from pydantic.alias_generators import to_camel RuntimeMode = Literal["pipeline", "realtime"] +VisionPromptMode = Literal["fixed", "runtime", "guided"] ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent"] AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"] TurnEndStrategy = Literal["silence", "smart_turn"] @@ -198,7 +199,8 @@ class AssistantUpsert(CamelModel): startup: StartupConfig = Field(default_factory=StartupConfig) vision_enabled: bool = False vision_model_resource_id: str | None = None - vision_focus_prompt: str = Field(default="", max_length=4000) + vision_prompt_mode: VisionPromptMode = "runtime" + vision_prompt: str = Field(default="", max_length=4000) model_resource_ids: dict[ModelType, str] = Field(default_factory=dict) knowledge_base_id: str | None = None knowledge_retrieval_config: KnowledgeRetrievalConfig = Field( @@ -264,15 +266,6 @@ class AssistantOut(AssistantUpsert): updated_at: str | None = None -class VisionFocusPromptGenerateRequest(CamelModel): - source_prompt: str = Field(min_length=1, max_length=20000) - model_resource_id: str = Field(min_length=1, max_length=40) - - -class VisionFocusPromptGenerateResult(CamelModel): - focus_prompt: str - - class DynamicVariableDefinition(CamelModel): type: DynamicVariableType = "string" required: bool = False diff --git a/backend/services/brains/workflow_brain.py b/backend/services/brains/workflow_brain.py index 1de265c..262c1ee 100644 --- a/backend/services/brains/workflow_brain.py +++ b/backend/services/brains/workflow_brain.py @@ -670,7 +670,10 @@ class WorkflowBrain(BaseBrain): append_function(self._flow_tool(tool, node_id)) append_function(self._knowledge_function(node_id)) if stage.vision_enabled and self._require_runtime().vision_function: - append_function(self._require_runtime().vision_function) + vision_function = self._require_runtime().vision_function + append_function( + vision_function(stage) if callable(vision_function) else vision_function + ) if self._engine.llm_routing_mode() == "edge_tool": for edge in self._engine.edge_tool_edges(node_id): append_function(self._edge_tool(edge, node_id)) diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 5028f2c..18632bd 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -301,7 +301,8 @@ async def resolve_runtime_config( ), vision_enabled=assistant.vision_enabled, vision_model_resource_id=assistant.vision_model_resource_id, - vision_focus_prompt=assistant.vision_focus_prompt or "", + vision_prompt_mode=assistant.vision_prompt_mode or "runtime", + vision_prompt=assistant.vision_prompt or "", vision_model=str(_value(vision_resource, "modelId", "")), vision_llm_interface_type=( vision_resource.interface_type if vision_resource else "openai-llm" diff --git a/backend/services/node_specs.py b/backend/services/node_specs.py index 1695690..edf1706 100644 --- a/backend/services/node_specs.py +++ b/backend/services/node_specs.py @@ -6,6 +6,7 @@ from collections import defaultdict, deque from copy import deepcopy from typing import Any +from models import VISION_PROMPT_MODES from services.message_policy import ( MESSAGE_COMPLETION_POLICIES, MESSAGE_CONFIRMATION, @@ -199,6 +200,11 @@ def _normalize_agent_data(data: dict[str, Any]) -> None: data["stateVariableNames"] = list( dict.fromkeys(str(name) for name in state_names or [] if str(name)) ) + legacy_prompt = str(data.pop("visionFocusPrompt", "") or "").strip() + if "visionPrompt" not in data: + data["visionPrompt"] = legacy_prompt + if "visionPromptMode" not in data: + data["visionPromptMode"] = "guided" if legacy_prompt else "runtime" if "inheritGlobalConfig" not in data: has_node_overrides = any( ( @@ -207,7 +213,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None: data.get("ttsResourceId"), data.get("visionEnabled"), data.get("visionModelResourceId"), - data.get("visionFocusPrompt"), + data.get("visionPrompt"), data.get("knowledgeBaseId"), data.get("toolIds"), ) @@ -268,7 +274,10 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") -> settings.setdefault("defaultTtsResourceId", "") settings.setdefault("visionEnabled", False) settings.setdefault("visionModelResourceId", "") - settings.setdefault("visionFocusPrompt", "") + legacy_prompt = str(settings.pop("visionFocusPrompt", "") or "").strip() + settings.setdefault("visionPrompt", legacy_prompt) + if "visionPromptMode" not in settings: + settings["visionPromptMode"] = "guided" if legacy_prompt else "runtime" settings.setdefault("toolIds", []) settings.setdefault("knowledgeBaseId", "") settings.setdefault("knowledgeMode", "automatic") @@ -471,6 +480,20 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: not isinstance(name, str) for name in state_names ): errors.append(f"Agent 节点 {node_id} 的状态变量授权必须是列表") + source = settings if data.get("inheritGlobalConfig", True) else data + vision_prompt_mode = str(source.get("visionPromptMode") or "runtime") + if vision_prompt_mode not in VISION_PROMPT_MODES: + errors.append( + f"Agent 节点 {node_id} 的视觉提示词模式无效:{vision_prompt_mode}" + ) + if ( + source.get("visionEnabled") + and vision_prompt_mode in {"fixed", "guided"} + and not str(source.get("visionPrompt") or "").strip() + ): + errors.append( + f"Agent 节点 {node_id} 的固定或引导模式必须填写视觉提示词" + ) if runtime_mode == "realtime": if data.get("contextPolicy", "inherit") != "inherit": errors.append( @@ -487,7 +510,6 @@ def validate_graph(graph: dict[str, Any]) -> list[str]: errors.append( f"Realtime Agent 节点 {node_id} 不支持节点级模型、语音或交互策略覆盖" ) - source = settings if data.get("inheritGlobalConfig", True) else data if ( source.get("knowledgeBaseId") and source.get("knowledgeMode") != "on_demand" diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index b4490be..988254b 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -38,8 +38,9 @@ from services.vision import ( image_data_uri, image_frame_from_jpeg, image_jpeg_bytes, + resolve_vision_question, vision_system_hint, - vision_tool_description, + vision_tool_schema_parts, ) from services.workflow_engine import WorkflowEngine @@ -349,7 +350,9 @@ async def run_pipeline( def with_vision_hint(text: str) -> str: hints = [] if vision_enabled and cfg.type != "workflow": - hints.append(vision_system_hint(cfg.vision_focus_prompt)) + hints.append( + vision_system_hint(cfg.vision_prompt_mode, cfg.vision_prompt) + ) if cfg.knowledge_base_id: hints.append( AUTOMATIC_KNOWLEDGE_SYSTEM_HINT @@ -450,7 +453,8 @@ async def run_pipeline( workflow_vision_scope: dict[str, Any] = { "enabled": False, "vision_model_resource_id": None, - "vision_focus_prompt": "", + "vision_prompt_mode": "runtime", + "vision_prompt": "", "llm_resource_id": None, } @@ -458,16 +462,15 @@ async def run_pipeline( if cfg.type == "workflow": return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope) return vision_native_mode + vision_description, vision_properties, vision_required = vision_tool_schema_parts( + cfg.vision_prompt_mode, + cfg.vision_prompt, + ) vision_schema = FunctionSchema( name=VISION_TOOL_NAME, - description=vision_tool_description(cfg.vision_focus_prompt), - properties={ - "question": { - "type": "string", - "description": "用户关于当前视频画面的具体问题。", - } - }, - required=["question"], + description=vision_description, + properties=vision_properties, + required=vision_required, ) knowledge_schema = FunctionSchema( name=KNOWLEDGE_TOOL_NAME, @@ -498,7 +501,15 @@ async def run_pipeline( await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"}) async def fetch_user_image(params: FunctionCallParams): - question = str(params.arguments.get("question") or "请描述当前画面。") + try: + question = resolve_vision_question( + cfg.vision_prompt_mode, + cfg.vision_prompt, + params.arguments, + ) + except ValueError as exc: + await params.result_callback({"status": "error", "message": str(exc)}) + return user_id = vision_state.get("client_id") if not user_id: await params.result_callback( @@ -576,14 +587,26 @@ async def run_pipeline( result = config_with_main_llm_as_vision(cfg) else: raise ValueError(f"视觉模型资源未加载:{vision_resource_id}") - result.vision_focus_prompt = str( - workflow_vision_scope.get("vision_focus_prompt") or "" + result.vision_prompt_mode = str( + workflow_vision_scope.get("vision_prompt_mode") or "runtime" + ) + result.vision_prompt = str( + workflow_vision_scope.get("vision_prompt") or "" ).strip() return result if cfg.type == "workflow" and vision_enabled: async def native_flow_fetch_user_image(params: FunctionCallParams) -> None: - question = str(params.arguments.get("question") or "请描述当前画面。") + try: + vision_cfg = active_vision_config() + question = resolve_vision_question( + vision_cfg.vision_prompt_mode, + vision_cfg.vision_prompt, + params.arguments, + ) + except ValueError as exc: + await params.result_callback({"status": "error", "message": str(exc)}) + return user_id = vision_state.get("client_id") if not user_id: await params.result_callback( @@ -609,7 +632,15 @@ async def run_pipeline( "status": "disabled", "message": "当前 Agent 节点未开启视觉理解。", } - question = str((args or {}).get("question") or "请描述当前画面。") + vision_cfg = active_vision_config() + try: + question = resolve_vision_question( + vision_cfg.vision_prompt_mode, + vision_cfg.vision_prompt, + args, + ) + except ValueError as exc: + return {"status": "error", "message": str(exc)} user_id = vision_state.get("client_id") if not user_id: return { @@ -625,7 +656,7 @@ async def run_pipeline( try: frame = await vision_capture.request_image(llm, request) observation = await analyze_image_with_vision_model( - active_vision_config(), + vision_cfg, frame, question, ) @@ -654,14 +685,19 @@ async def run_pipeline( active_vision_uses_main_llm, ) - workflow_vision_function = FlowsFunctionSchema( - name=VISION_TOOL_NAME, - description=vision_schema.description, - properties=vision_schema.properties, - required=vision_schema.required, - handler=flow_fetch_user_image, - cancel_on_interruption=True, - ) + def workflow_vision_function(stage) -> FlowsFunctionSchema: + description, properties, required = vision_tool_schema_parts( + stage.vision_prompt_mode, + stage.vision_prompt, + ) + return FlowsFunctionSchema( + name=VISION_TOOL_NAME, + description=description, + properties=properties, + required=required, + handler=flow_fetch_user_image, + cancel_on_interruption=True, + ) if vision_enabled and cfg.type != "workflow": llm.register_function(VISION_TOOL_NAME, fetch_user_image) diff --git a/backend/services/vision.py b/backend/services/vision.py index 0a56224..cdd49de 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -5,8 +5,14 @@ from __future__ import annotations import asyncio import base64 from io import BytesIO +from typing import Any -from models import AssistantConfig, RuntimeModelResource +from models import ( + VISION_PROMPT_MODES, + AssistantConfig, + RuntimeModelResource, + VisionPromptMode, +) from openai import AsyncOpenAI from PIL import Image from pipecat.frames.frames import UserImageRawFrame @@ -22,12 +28,6 @@ VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断，请明确说明不确定。" ) -VISION_FOCUS_GENERATION_SYSTEM_PROMPT = ( - "你负责把对话助手提示词整理成一段可编辑的视觉关注目标。" - "只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；" - "不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、" - "意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。" -) def _require(value: str, label: str) -> str: @@ -36,83 +36,96 @@ def _require(value: str, label: str) -> str: raise ValueError(f"缺少模型资源配置: {label}") -def vision_system_hint(focus_prompt: str = "") -> str: - """Describe when to use vision and keep a configured business focus visible.""" +def normalize_vision_prompt_mode(value: object) -> VisionPromptMode: + """Return a supported mode while keeping older saved configs compatible.""" - focus = focus_prompt.strip() - if not focus: - return VISION_SYSTEM_HINT + mode = str(value or "runtime") + if mode == "fixed": + return "fixed" + if mode == "guided": + return "guided" + return "runtime" + + +def _compact_prompt(prompt: str, *, limit: int = 4000) -> str: + return " ".join(prompt.split())[:limit] + + +def vision_system_hint( + mode: VisionPromptMode = "runtime", + configured_prompt: str = "", +) -> str: + """Tell the conversation model how the camera tool forms its question.""" + + prompt_mode = normalize_vision_prompt_mode(mode) + prompt = _compact_prompt(configured_prompt) + if prompt_mode == "fixed": + return ( + f"{VISION_SYSTEM_HINT}\n" + f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。" + ) + if prompt_mode == "guided": + return ( + f"{VISION_SYSTEM_HINT}\n" + "调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导：" + f"{prompt}" + ) return ( f"{VISION_SYSTEM_HINT}\n" - f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}" + "调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。" ) -def vision_analysis_system_prompt(focus_prompt: str = "") -> str: - """Keep the safety baseline fixed while appending an editable business focus.""" +def vision_tool_schema_parts( + mode: VisionPromptMode = "runtime", + configured_prompt: str = "", +) -> tuple[str, dict[str, Any], list[str]]: + """Build the tool contract for one configured visual-question mode.""" - focus = focus_prompt.strip() - if not focus: - return VISION_ANALYSIS_SYSTEM_PROMPT - return ( - f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n" - f"[业务视觉关注目标]\n{focus}\n" - "该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。" - ) - - -def vision_tool_description(focus_prompt: str = "") -> str: base = ( "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" "人/物品/环境状态或需要视觉判断时调用。" ) - focus = " ".join(focus_prompt.split()) - return f"{base} 当前业务重点：{focus[:600]}" if focus else base - - -async def generate_vision_focus_prompt( - resource: RuntimeModelResource, - source_prompt: str, -) -> str: - """Generate one editable focus draft with an existing LLM resource.""" - - if resource.interface_type not in {"openai-llm", "dashscope-llm"}: - raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}") - source = source_prompt.strip() - if not source: - raise ValueError("助手提示词不能为空") - values = resource.values or {} - secrets = resource.secrets or {} - extra_body = values.get("extraBody") - extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} - client = AsyncOpenAI( - api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"), - base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"), - ) - try: - response = await client.chat.completions.create( - model=_require(str(values.get("modelId") or ""), "LLM modelId"), - messages=[ - {"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT}, - {"role": "user", "content": source}, - ], - **extra, + prompt_mode = normalize_vision_prompt_mode(mode) + prompt = _compact_prompt(configured_prompt, limit=600) + if prompt_mode == "fixed": + return ( + f"{base} 每次都检查同一个固定问题：{prompt}", + {}, + [], ) - finally: - await client.close() - content = response.choices[0].message.content if response.choices else "" - result = ( - content.strip() - if isinstance(content, str) - else str(content or "").strip() + + question_description = "结合当前对话生成的、关于当前视频画面的具体问题。" + description = base + if prompt_mode == "guided": + description = f"{base} 生成问题时必须遵守固定引导：{prompt}" + question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}" + return ( + description, + {"question": {"type": "string", "description": question_description}}, + ["question"], ) - if not result: - raise ValueError("模型没有生成有效的视觉关注目标") - if result.startswith("```") and result.endswith("```"): - result = result[3:-3].strip() - if result.startswith("text"): - result = result[4:].strip() - return result[:4000] + + +def resolve_vision_question( + mode: VisionPromptMode, + configured_prompt: str, + arguments: dict[str, Any] | None, +) -> str: + """Resolve the exact question sent with the captured image.""" + + prompt_mode = normalize_vision_prompt_mode(mode) + if prompt_mode == "fixed": + question = configured_prompt.strip() + else: + question = str((arguments or {}).get("question") or "").strip() + if not question: + raise ValueError( + "固定视觉提示词为空" + if prompt_mode == "fixed" + else "视觉问题为空" + ) + return question def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes: @@ -173,9 +186,7 @@ async def analyze_image_with_vision_model( messages=[ { "role": "system", - "content": vision_analysis_system_prompt( - cfg.vision_focus_prompt - ), + "content": VISION_ANALYSIS_SYSTEM_PROMPT, }, { "role": "user", diff --git a/backend/services/workflow/agent.py b/backend/services/workflow/agent.py index b12358d..44e2a7d 100644 --- a/backend/services/workflow/agent.py +++ b/backend/services/workflow/agent.py @@ -49,7 +49,7 @@ class WorkflowAgentStage: if stage.vision_enabled: stage_prompt = ( f"{stage_prompt}\n\n[视觉能力]\n" - f"{vision_system_hint(stage.vision_focus_prompt)}" + f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}" ) instruction = ( EDGE_TOOL_STAGE_INSTRUCTION @@ -112,7 +112,8 @@ class WorkflowAgentStage: { "enabled": stage.vision_enabled, "vision_model_resource_id": stage.vision_model_resource_id, - "vision_focus_prompt": stage.vision_focus_prompt, + "vision_prompt_mode": stage.vision_prompt_mode, + "vision_prompt": stage.vision_prompt, "llm_resource_id": stage.llm_resource_id, } ) diff --git a/backend/services/workflow/realtime.py b/backend/services/workflow/realtime.py index 78944de..08267e7 100644 --- a/backend/services/workflow/realtime.py +++ b/backend/services/workflow/realtime.py @@ -34,8 +34,9 @@ from services.vision import ( VISION_TOOL_NAME, analyze_image_with_vision_model, config_with_vision_resource, + resolve_vision_question, vision_system_hint, - vision_tool_description, + vision_tool_schema_parts, ) from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION from services.workflow.models import WorkflowRuntimeState, WorkflowStatus @@ -414,7 +415,10 @@ class WorkflowRealtimeController: f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}" ) if stage.vision_enabled: - prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}" + prompt = ( + f"{prompt}\n" + f"{vision_system_hint(stage.vision_prompt_mode, stage.vision_prompt)}" + ) return prompt def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]: @@ -573,10 +577,15 @@ class WorkflowRealtimeController: {"status": "stale", "message": "当前 Agent 已经切换。"}, continue_response=False, ) - question = str(arguments.get("question") or "").strip() - if not question: + try: + question = resolve_vision_question( + stage.vision_prompt_mode, + stage.vision_prompt, + arguments, + ) + except ValueError as exc: return RealtimeToolResult( - {"status": "error", "message": "视觉问题为空"} + {"status": "error", "message": str(exc)} ) if self._runtime.capture_image is None: return RealtimeToolResult( @@ -605,7 +614,6 @@ class WorkflowRealtimeController: try: if vision_cfg is not None: - vision_cfg.vision_focus_prompt = stage.vision_focus_prompt observation = await analyze_image_with_vision_model( vision_cfg, frame, @@ -634,17 +642,16 @@ class WorkflowRealtimeController: } ) + description, properties, required = vision_tool_schema_parts( + stage.vision_prompt_mode, + stage.vision_prompt, + ) return ( RealtimeTool( name=VISION_TOOL_NAME, - description=vision_tool_description(stage.vision_focus_prompt), - properties={ - "question": { - "type": "string", - "description": "需要根据当前画面判断的具体问题", - } - }, - required=("question",), + description=description, + properties=properties, + required=tuple(required), ), handler, ) diff --git a/backend/services/workflow_engine.py b/backend/services/workflow_engine.py index 8e12316..261dbf9 100644 --- a/backend/services/workflow_engine.py +++ b/backend/services/workflow_engine.py @@ -23,7 +23,8 @@ class AgentStageConfig: tts_resource_id: str vision_enabled: bool vision_model_resource_id: str | None - vision_focus_prompt: str + vision_prompt_mode: str + vision_prompt: str tool_ids: tuple[str, ...] state_variable_names: tuple[str, ...] knowledge_base_id: str | None @@ -152,7 +153,8 @@ class WorkflowEngine: vision_model_resource_id=( str(source.get("visionModelResourceId") or "") or None ), - vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(), + vision_prompt_mode=str(source.get("visionPromptMode") or "runtime"), + vision_prompt=str(source.get("visionPrompt") or "").strip(), tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []), # Variable write scope always belongs to the Agent node, even when # its selected tool resources come from Workflow defaults. diff --git a/backend/tests/test_brains.py b/backend/tests/test_brains.py index 0d89aa7..ea70351 100644 --- a/backend/tests/test_brains.py +++ b/backend/tests/test_brains.py @@ -3,7 +3,7 @@ from __future__ import annotations import asyncio import unittest from types import SimpleNamespace -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch from models import AssistantConfig, RuntimeTool from pipecat.flows import FlowManager @@ -1782,7 +1782,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): } ) scopes = [] - vision_function = object() + vision_function = MagicMock(side_effect=lambda _stage: object()) async def queue_frame(_frame): pass @@ -1800,13 +1800,18 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): await brain._apply_agent_stage("agent") inherited_config = brain._agent_config("agent") - self.assertIn(vision_function, inherited_config["functions"]) + self.assertEqual(len(inherited_config["functions"]), 1) + vision_function.assert_called_once() + selected_stage = vision_function.call_args.args[0] + self.assertEqual(selected_stage.vision_prompt_mode, "runtime") self.assertIn("fetch_user_image", inherited_config["role_message"]) self.assertEqual( scopes[-1], { "enabled": True, "vision_model_resource_id": "vision_global", + "vision_prompt_mode": "runtime", + "vision_prompt": "", "llm_resource_id": "llm_global", }, ) @@ -1821,7 +1826,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase): ) await brain._apply_agent_stage("agent") custom_config = brain._agent_config("agent") - self.assertNotIn(vision_function, custom_config["functions"]) + self.assertEqual(custom_config["functions"], []) self.assertNotIn("fetch_user_image", custom_config["role_message"]) self.assertFalse(scopes[-1]["enabled"]) diff --git a/backend/tests/test_vision_focus.py b/backend/tests/test_vision_focus.py deleted file mode 100644 index fdb5a4c..0000000 --- a/backend/tests/test_vision_focus.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import unittest -from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock, patch - -from models import RuntimeModelResource -from services.vision import ( - VISION_ANALYSIS_SYSTEM_PROMPT, - VISION_SYSTEM_HINT, - generate_vision_focus_prompt, - vision_analysis_system_prompt, - vision_system_hint, - vision_tool_description, -) -from services.workflow_engine import WorkflowEngine - - -class VisionFocusPromptTest(unittest.TestCase): - def test_focus_extends_but_does_not_replace_safety_prompts(self): - focus = "优先识别车辆数量和人员状态，只报告可确认的信息。" - - system_hint = vision_system_hint(focus) - analysis_prompt = vision_analysis_system_prompt(focus) - - self.assertIn(VISION_SYSTEM_HINT, system_hint) - self.assertIn(focus, system_hint) - self.assertIn(VISION_ANALYSIS_SYSTEM_PROMPT, analysis_prompt) - self.assertIn(focus, analysis_prompt) - self.assertIn("优先回答当前用户问题", analysis_prompt) - - def test_empty_focus_preserves_previous_runtime_prompts(self): - self.assertEqual(vision_system_hint(), VISION_SYSTEM_HINT) - self.assertEqual( - vision_analysis_system_prompt(), - VISION_ANALYSIS_SYSTEM_PROMPT, - ) - - def test_tool_description_compacts_and_limits_focus(self): - description = vision_tool_description("车辆\n 数量 " * 300) - - self.assertIn("当前业务重点", description) - self.assertNotIn("\n", description) - self.assertLess(len(description), 700) - - def test_workflow_resolves_global_and_node_focus(self): - graph = { - "specVersion": 3, - "settings": { - "visionEnabled": True, - "visionFocusPrompt": "全局关注目标", - }, - "nodes": [ - { - "id": "global-agent", - "type": "agent", - "data": {"name": "全局 Agent", "prompt": "处理任务"}, - }, - { - "id": "custom-agent", - "type": "agent", - "data": { - "name": "独立 Agent", - "prompt": "检查车辆", - "inheritGlobalConfig": False, - "visionEnabled": True, - "visionFocusPrompt": "节点关注目标", - }, - }, - ], - "edges": [], - } - engine = WorkflowEngine(graph) - - self.assertEqual( - engine.agent_stage_config("global-agent").vision_focus_prompt, - "全局关注目标", - ) - self.assertEqual( - engine.agent_stage_config("custom-agent").vision_focus_prompt, - "节点关注目标", - ) - - -class VisionFocusGenerationTest(unittest.IsolatedAsyncioTestCase): - async def test_generates_editable_focus_with_selected_resource(self): - resource = RuntimeModelResource( - id="llm_1", - name="生成模型", - capability="LLM", - interface_type="openai-llm", - values={ - "apiUrl": "https://llm.example/v1", - "modelId": "focus-model", - }, - secrets={"apiKey": "secret"}, - ) - client = MagicMock() - client.chat.completions.create = AsyncMock( - return_value=SimpleNamespace( - choices=[ - SimpleNamespace( - message=SimpleNamespace( - content="优先识别车辆数量、损伤位置和人员状态。" - ) - ) - ] - ) - ) - client.close = AsyncMock() - - with patch("services.vision.AsyncOpenAI", return_value=client): - result = await generate_vision_focus_prompt( - resource, - "你是交通事故处理助手，需要了解车辆和人员情况。", - ) - - self.assertEqual(result, "优先识别车辆数量、损伤位置和人员状态。") - request = client.chat.completions.create.await_args.kwargs - self.assertEqual(request["model"], "focus-model") - self.assertIn("交通事故处理助手", request["messages"][1]["content"]) - client.close.assert_awaited_once() - - -if __name__ == "__main__": - unittest.main() diff --git a/backend/tests/test_vision_prompt_modes.py b/backend/tests/test_vision_prompt_modes.py new file mode 100644 index 0000000..f60c3d8 --- /dev/null +++ b/backend/tests/test_vision_prompt_modes.py @@ -0,0 +1,227 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace + +from models import AssistantConfig +from services.brains.base import RealtimeBrainRuntime +from services.node_specs import normalize_graph, validate_graph +from services.vision import ( + VISION_SYSTEM_HINT, + resolve_vision_question, + vision_system_hint, + vision_tool_schema_parts, +) +from services.workflow_engine import WorkflowEngine +from services.workflow.realtime import WorkflowRealtimeController + + +class VisionPromptModeTest(unittest.TestCase): + def test_fixed_mode_has_no_runtime_question_parameter(self): + description, properties, required = vision_tool_schema_parts( + "fixed", + "识别车辆数量和人员状态", + ) + + self.assertIn("识别车辆数量和人员状态", description) + self.assertEqual(properties, {}) + self.assertEqual(required, []) + self.assertEqual( + resolve_vision_question("fixed", " 识别车辆数量 ", {}), + "识别车辆数量", + ) + + def test_runtime_mode_requires_model_generated_question(self): + description, properties, required = vision_tool_schema_parts("runtime") + + self.assertNotIn("固定引导", description) + self.assertIn("question", properties) + self.assertEqual(required, ["question"]) + self.assertEqual( + resolve_vision_question( + "runtime", + "不会使用这个配置", + {"question": "用户手里拿着什么？"}, + ), + "用户手里拿着什么？", + ) + + def test_guided_mode_exposes_guide_but_uses_runtime_question(self): + guide = "只询问画面中可直接确认的车辆损伤" + description, properties, required = vision_tool_schema_parts( + "guided", + guide, + ) + + self.assertIn(guide, description) + self.assertIn(guide, properties["question"]["description"]) + self.assertEqual(required, ["question"]) + self.assertEqual( + resolve_vision_question( + "guided", + guide, + {"question": "右侧车门是否有明显凹陷？"}, + ), + "右侧车门是否有明显凹陷？", + ) + + def test_mode_specific_system_hints(self): + self.assertIn(VISION_SYSTEM_HINT, vision_system_hint("runtime")) + self.assertIn("不需要生成", vision_system_hint("fixed", "固定问题")) + self.assertIn("固定引导", vision_system_hint("guided", "业务引导")) + + def test_missing_required_config_or_question_is_rejected(self): + with self.assertRaisesRegex(ValueError, "固定视觉提示词为空"): + resolve_vision_question("fixed", "", {}) + with self.assertRaisesRegex(ValueError, "视觉问题为空"): + resolve_vision_question("runtime", "", {}) + + +class WorkflowVisionPromptModeTest(unittest.TestCase): + def test_workflow_resolves_global_and_node_modes(self): + graph = { + "specVersion": 3, + "settings": { + "visionEnabled": True, + "visionPromptMode": "fixed", + "visionPrompt": "全局固定问题", + }, + "nodes": [ + { + "id": "global-agent", + "type": "agent", + "data": {"name": "全局 Agent", "prompt": "处理任务"}, + }, + { + "id": "custom-agent", + "type": "agent", + "data": { + "name": "独立 Agent", + "prompt": "检查车辆", + "inheritGlobalConfig": False, + "visionEnabled": True, + "visionPromptMode": "guided", + "visionPrompt": "节点引导", + }, + }, + ], + "edges": [], + } + engine = WorkflowEngine(graph) + + global_stage = engine.agent_stage_config("global-agent") + self.assertEqual(global_stage.vision_prompt_mode, "fixed") + self.assertEqual(global_stage.vision_prompt, "全局固定问题") + custom_stage = engine.agent_stage_config("custom-agent") + self.assertEqual(custom_stage.vision_prompt_mode, "guided") + self.assertEqual(custom_stage.vision_prompt, "节点引导") + + def test_legacy_focus_is_migrated_to_guided_mode(self): + normalized = normalize_graph( + { + "specVersion": 3, + "settings": {"visionFocusPrompt": "旧全局关注目标"}, + "nodes": [], + "edges": [], + } + ) + + self.assertEqual(normalized["settings"]["visionPromptMode"], "guided") + self.assertEqual(normalized["settings"]["visionPrompt"], "旧全局关注目标") + self.assertNotIn("visionFocusPrompt", normalized["settings"]) + + def test_fixed_or_guided_mode_requires_prompt_when_vision_is_enabled(self): + graph = { + "specVersion": 3, + "settings": { + "visionEnabled": True, + "visionPromptMode": "fixed", + "visionPrompt": "", + }, + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": {"name": "Agent", "prompt": "处理任务"}, + } + ], + "edges": [], + } + + self.assertTrue( + any("必须填写视觉提示词" in error for error in validate_graph(graph)) + ) + + def test_invalid_mode_is_reported_instead_of_silently_rewritten(self): + graph = { + "specVersion": 3, + "settings": {"visionPromptMode": "unexpected"}, + "nodes": [ + { + "id": "agent", + "type": "agent", + "data": {"name": "Agent", "prompt": "处理任务"}, + } + ], + "edges": [], + } + + self.assertTrue( + any("视觉提示词模式无效" in error for error in validate_graph(graph)) + ) + + def test_realtime_tool_contract_changes_with_mode(self): + graph = { + "specVersion": 3, + "settings": {}, + "nodes": [ + { + "id": "fixed-agent", + "type": "agent", + "data": { + "name": "固定", + "prompt": "处理任务", + "inheritGlobalConfig": False, + "visionEnabled": True, + "visionPromptMode": "fixed", + "visionPrompt": "检查安全帽", + }, + }, + { + "id": "guided-agent", + "type": "agent", + "data": { + "name": "引导", + "prompt": "处理任务", + "inheritGlobalConfig": False, + "visionEnabled": True, + "visionPromptMode": "guided", + "visionPrompt": "只询问安全设备", + }, + }, + ], + "edges": [], + } + engine = WorkflowEngine(graph) + controller = WorkflowRealtimeController( + cfg=AssistantConfig(type="workflow"), + engine=engine, + store=SimpleNamespace(), + runtime=RealtimeBrainRuntime( + realtime=SimpleNamespace(), + queue_frame=lambda _frame: None, + call_end=SimpleNamespace(ending=False), + ), + ) + + fixed_tool, _fixed_handler = controller._vision_tool("fixed-agent", 0) + guided_tool, _guided_handler = controller._vision_tool("guided-agent", 0) + + self.assertEqual(fixed_tool.properties, {}) + self.assertEqual(fixed_tool.required, ()) + self.assertIn("只询问安全设备", guided_tool.description) + self.assertEqual(guided_tool.required, ("question",)) + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/components/assistant-editor/prompt-editor.tsx b/frontend/src/components/assistant-editor/prompt-editor.tsx index b6bfb5c..3aad902 100644 --- a/frontend/src/components/assistant-editor/prompt-editor.tsx +++ b/frontend/src/components/assistant-editor/prompt-editor.tsx @@ -648,17 +648,19 @@ export function PromptEditor({ hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。" enabled={form.visionEnabled} modelResourceId={form.visionModelResourceId} - focusPrompt={form.visionFocusPrompt} - sourcePrompt={form.prompt} - generatorModelResourceId={form.model} + promptMode={form.visionPromptMode} + prompt={form.visionPrompt} mainModelResourceId={form.model} modelOptions={visionModelOptions} onEnabledChange={handlePromptVisionEnabledChange} onModelResourceIdChange={(value) => updateForm("visionModelResourceId", value) } - onFocusPromptChange={(value) => - updateForm("visionFocusPrompt", value) + onPromptModeChange={(value) => + updateForm("visionPromptMode", value) + } + onPromptChange={(value) => + updateForm("visionPrompt", value) } /> )} diff --git a/frontend/src/components/assistant-editor/types.ts b/frontend/src/components/assistant-editor/types.ts index ae9084e..99d4dee 100644 --- a/frontend/src/components/assistant-editor/types.ts +++ b/frontend/src/components/assistant-editor/types.ts @@ -4,6 +4,7 @@ import type { KnowledgeRetrievalConfig, StartupConfig, TurnConfig, + VisionPromptMode, } from "@/lib/api"; export type RuntimeMode = "pipeline" | "realtime"; @@ -26,6 +27,7 @@ export type AssistantForm = { startup: StartupConfig; visionEnabled: boolean; visionModelResourceId: string; - visionFocusPrompt: string; + visionPromptMode: VisionPromptMode; + visionPrompt: string; toolIds: string[]; }; diff --git a/frontend/src/components/editor/vision-config-section.tsx b/frontend/src/components/editor/vision-config-section.tsx index 6ddd66a..1aa60c5 100644 --- a/frontend/src/components/editor/vision-config-section.tsx +++ b/frontend/src/components/editor/vision-config-section.tsx @@ -1,53 +1,71 @@ "use client"; -import { useState } from "react"; -import { Camera, Loader2, Sparkles } from "lucide-react"; +import { Camera, Check } from "lucide-react"; import { ResourceSelectField, ToggleRow, } from "@/components/assistant-editor/editor-controls"; import { SectionCard } from "@/components/editor/section-card"; -import { Button } from "@/components/ui/button"; import { Textarea } from "@/components/ui/textarea"; -import { assistantsApi } from "@/lib/api"; +import type { VisionPromptMode } from "@/lib/api"; type VisionModelOption = { value: string; label: string; }; +const promptModes: Array<{ + value: VisionPromptMode; + title: string; + description: string; +}> = [ + { + value: "fixed", + title: "固定提示词", + description: "每次获取画面都发送同一个问题。", + }, + { + value: "runtime", + title: "运行时自动生成", + description: "模型结合当前对话，临时生成看图问题。", + }, + { + value: "guided", + title: "固定提示词引导", + description: "模型动态生成问题，同时遵守固定引导。", + }, +]; + export function VisionConfigSection({ description, hint, enabled, modelResourceId, - focusPrompt, - sourcePrompt, - generatorModelResourceId, + promptMode, + prompt, mainModelResourceId, mainModelSupportsImageInput, modelOptions, onEnabledChange, onModelResourceIdChange, - onFocusPromptChange, + onPromptModeChange, + onPromptChange, }: { description: string; hint: string; enabled: boolean; modelResourceId: string; - focusPrompt: string; - sourcePrompt: string; - generatorModelResourceId: string; + promptMode: VisionPromptMode; + prompt: string; mainModelResourceId: string; mainModelSupportsImageInput?: boolean; modelOptions: VisionModelOption[]; onEnabledChange: (enabled: boolean) => void; onModelResourceIdChange: (modelResourceId: string) => void; - onFocusPromptChange: (focusPrompt: string) => void; + onPromptModeChange: (promptMode: VisionPromptMode) => void; + onPromptChange: (prompt: string) => void; }) { - const [generating, setGenerating] = useState(false); - const [generationError, setGenerationError] = useState(""); const mainModelSupportsVision = mainModelSupportsImageInput ?? modelOptions.some((option) => option.value === mainModelResourceId); @@ -55,25 +73,6 @@ export function VisionConfigSection({ (option) => option.value !== mainModelResourceId, ); - async function generateFocusPrompt() { - if (!sourcePrompt.trim() || !generatorModelResourceId || generating) return; - setGenerating(true); - setGenerationError(""); - try { - const result = await assistantsApi.generateVisionFocus({ - sourcePrompt, - modelResourceId: generatorModelResourceId, - }); - onFocusPromptChange(result.focusPrompt); - } catch (error) { - setGenerationError( - error instanceof Error ? error.message : "生成视觉关注目标失败", - ); - } finally { - setGenerating(false); - } - } - return ( } @@ -100,56 +99,79 @@ export function VisionConfigSection({ 当前模型未标记支持图片输入，请选择独立视觉模型。

)} -
-
- - +
+ +
+ {promptModes.map((option) => { + const selected = promptMode === option.value; + return ( + + ); + })}
-