feat: add configurable vision focus prompts

backend/services/config_resolver.py

@@ -301,6 +301,7 @@ async def resolve_runtime_config(
         ),
         vision_enabled=assistant.vision_enabled,
         vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_focus_prompt=assistant.vision_focus_prompt or "",
         vision_model=str(_value(vision_resource, "modelId", "")),
         vision_llm_interface_type=(
             vision_resource.interface_type if vision_resource else "openai-llm"

backend/services/node_specs.py

@@ -207,6 +207,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
                 data.get("ttsResourceId"),
                 data.get("visionEnabled"),
                 data.get("visionModelResourceId"),
+                data.get("visionFocusPrompt"),
                 data.get("knowledgeBaseId"),
                 data.get("toolIds"),
             )
@@ -267,6 +268,7 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
     settings.setdefault("defaultTtsResourceId", "")
     settings.setdefault("visionEnabled", False)
     settings.setdefault("visionModelResourceId", "")
+    settings.setdefault("visionFocusPrompt", "")
     settings.setdefault("toolIds", [])
     settings.setdefault("knowledgeBaseId", "")
     settings.setdefault("knowledgeMode", "automatic")

backend/services/pipecat/pipeline.py

@@ -31,7 +31,6 @@ from services.input_assets import consume_input_image
 from services.client_tools import ClientToolBroker
 from services.tool_policy import policy_for_tool
 from services.vision import (
-    VISION_SYSTEM_HINT,
     VISION_TOOL_NAME,
     analyze_image_with_vision_model,
     config_with_main_llm_as_vision,
@@ -39,6 +38,8 @@ from services.vision import (
     image_data_uri,
     image_frame_from_jpeg,
     image_jpeg_bytes,
+    vision_system_hint,
+    vision_tool_description,
 )
 from services.workflow_engine import WorkflowEngine
 
@@ -348,7 +349,7 @@ async def run_pipeline(
     def with_vision_hint(text: str) -> str:
         hints = []
         if vision_enabled and cfg.type != "workflow":
-            hints.append(VISION_SYSTEM_HINT)
+            hints.append(vision_system_hint(cfg.vision_focus_prompt))
         if cfg.knowledge_base_id:
             hints.append(
                 AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
@@ -449,6 +450,7 @@ async def run_pipeline(
     workflow_vision_scope: dict[str, Any] = {
         "enabled": False,
         "vision_model_resource_id": None,
+        "vision_focus_prompt": "",
         "llm_resource_id": None,
     }
 
@@ -458,10 +460,7 @@ async def run_pipeline(
         return vision_native_mode
     vision_schema = FunctionSchema(
         name=VISION_TOOL_NAME,
-        description=(
-            "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
-            "人/物品/环境状态或需要视觉判断时调用。"
-        ),
+        description=vision_tool_description(cfg.vision_focus_prompt),
         properties={
             "question": {
                 "type": "string",
@@ -572,10 +571,15 @@ async def run_pipeline(
         resource_id = vision_resource_id or llm_resource_id
         resource = cfg.workflow_model_resources.get(resource_id)
         if resource:
-            return config_with_vision_resource(cfg, resource)
-        if not vision_resource_id:
-            return config_with_main_llm_as_vision(cfg)
-        raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
+            result = config_with_vision_resource(cfg, resource)
+        elif not vision_resource_id:
+            result = config_with_main_llm_as_vision(cfg)
+        else:
+            raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
+        result.vision_focus_prompt = str(
+            workflow_vision_scope.get("vision_focus_prompt") or ""
+        ).strip()
+        return result
 
     if cfg.type == "workflow" and vision_enabled:
         async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:

backend/services/vision.py

@@ -22,6 +22,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
     "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
     "如果画面不足以判断，请明确说明不确定。"
 )
+VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
+    "你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
+    "只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
+    "不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
+    "意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
+)
 
 
 def _require(value: str, label: str) -> str:
@@ -30,6 +36,85 @@ def _require(value: str, label: str) -> str:
     raise ValueError(f"缺少模型资源配置: {label}")
 
 
+def vision_system_hint(focus_prompt: str = "") -> str:
+    """Describe when to use vision and keep a configured business focus visible."""
+
+    focus = focus_prompt.strip()
+    if not focus:
+        return VISION_SYSTEM_HINT
+    return (
+        f"{VISION_SYSTEM_HINT}\n"
+        f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
+    )
+
+
+def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
+    """Keep the safety baseline fixed while appending an editable business focus."""
+
+    focus = focus_prompt.strip()
+    if not focus:
+        return VISION_ANALYSIS_SYSTEM_PROMPT
+    return (
+        f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
+        f"[业务视觉关注目标]\n{focus}\n"
+        "该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
+    )
+
+
+def vision_tool_description(focus_prompt: str = "") -> str:
+    base = (
+        "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
+        "人/物品/环境状态或需要视觉判断时调用。"
+    )
+    focus = " ".join(focus_prompt.split())
+    return f"{base} 当前业务重点：{focus[:600]}" if focus else base
+
+
+async def generate_vision_focus_prompt(
+    resource: RuntimeModelResource,
+    source_prompt: str,
+) -> str:
+    """Generate one editable focus draft with an existing LLM resource."""
+
+    if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
+        raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
+    source = source_prompt.strip()
+    if not source:
+        raise ValueError("助手提示词不能为空")
+    values = resource.values or {}
+    secrets = resource.secrets or {}
+    extra_body = values.get("extraBody")
+    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
+    client = AsyncOpenAI(
+        api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
+        base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
+    )
+    try:
+        response = await client.chat.completions.create(
+            model=_require(str(values.get("modelId") or ""), "LLM modelId"),
+            messages=[
+                {"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
+                {"role": "user", "content": source},
+            ],
+            **extra,
+        )
+    finally:
+        await client.close()
+    content = response.choices[0].message.content if response.choices else ""
+    result = (
+        content.strip()
+        if isinstance(content, str)
+        else str(content or "").strip()
+    )
+    if not result:
+        raise ValueError("模型没有生成有效的视觉关注目标")
+    if result.startswith("```") and result.endswith("```"):
+        result = result[3:-3].strip()
+        if result.startswith("text"):
+            result = result[4:].strip()
+    return result[:4000]
+
+
 def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
     """Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
     if not frame.format:
@@ -86,7 +171,12 @@ async def analyze_image_with_vision_model(
         response = await client.chat.completions.create(
             model=_require(cfg.vision_model, "Vision LLM modelId"),
             messages=[
-                {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
+                {
+                    "role": "system",
+                    "content": vision_analysis_system_prompt(
+                        cfg.vision_focus_prompt
+                    ),
+                },
                 {
                     "role": "user",
                     "content": [

backend/services/workflow/agent.py

@@ -11,7 +11,7 @@ from pipecat.services.settings import LLMSettings
 
 from services.brains.base import BrainRuntime
 from services.runtime_variables import DynamicVariableStore
-from services.vision import VISION_SYSTEM_HINT
+from services.vision import vision_system_hint
 from services.workflow_engine import WorkflowEngine
 from services.workflow_router import WorkflowLLMRouter
 
@@ -47,7 +47,10 @@ class WorkflowAgentStage:
         stage_prompt = self._engine.prompt_for(node_id, self._store)
         stage = self._engine.agent_stage_config(node_id)
         if stage.vision_enabled:
-            stage_prompt = f"{stage_prompt}\n\n[视觉能力]\n{VISION_SYSTEM_HINT}"
+            stage_prompt = (
+                f"{stage_prompt}\n\n[视觉能力]\n"
+                f"{vision_system_hint(stage.vision_focus_prompt)}"
+            )
         instruction = (
             EDGE_TOOL_STAGE_INSTRUCTION
             if self._engine.llm_routing_mode() == "edge_tool"
@@ -109,6 +112,7 @@ class WorkflowAgentStage:
                 {
                     "enabled": stage.vision_enabled,
                     "vision_model_resource_id": stage.vision_model_resource_id,
+                    "vision_focus_prompt": stage.vision_focus_prompt,
                     "llm_resource_id": stage.llm_resource_id,
                 }
             )

backend/services/workflow/realtime.py

@@ -31,10 +31,11 @@ from services.system_tools import state_update_properties, system_tool_kind
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 from services.vision import (
-    VISION_SYSTEM_HINT,
     VISION_TOOL_NAME,
     analyze_image_with_vision_model,
     config_with_vision_resource,
+    vision_system_hint,
+    vision_tool_description,
 )
 from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
 from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
@@ -407,12 +408,13 @@ class WorkflowRealtimeController:
         return RealtimeActivation(continue_response=generate)
 
     def _agent_prompt(self, node_id: str) -> str:
+        stage = self._engine.agent_stage_config(node_id)
         prompt = (
             f"{self._engine.prompt_for(node_id, self._store)}\n\n"
             f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
         )
-        if self._engine.agent_stage_config(node_id).vision_enabled:
-            prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
+        if stage.vision_enabled:
+            prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
         return prompt
 
     def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
@@ -603,6 +605,7 @@ class WorkflowRealtimeController:
 
             try:
                 if vision_cfg is not None:
+                    vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
                     observation = await analyze_image_with_vision_model(
                         vision_cfg,
                         frame,
@@ -634,7 +637,7 @@ class WorkflowRealtimeController:
         return (
             RealtimeTool(
                 name=VISION_TOOL_NAME,
-                description="获取用户摄像头的当前画面，并回答一个视觉问题。",
+                description=vision_tool_description(stage.vision_focus_prompt),
                 properties={
                     "question": {
                         "type": "string",

backend/services/workflow_engine.py

@@ -23,6 +23,7 @@ class AgentStageConfig:
     tts_resource_id: str
     vision_enabled: bool
     vision_model_resource_id: str | None
+    vision_focus_prompt: str
     tool_ids: tuple[str, ...]
     state_variable_names: tuple[str, ...]
     knowledge_base_id: str | None
@@ -151,6 +152,7 @@ class WorkflowEngine:
             vision_model_resource_id=(
                 str(source.get("visionModelResourceId") or "") or None
             ),
+            vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(),
             tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []),
             # Variable write scope always belongs to the Agent node, even when
             # its selected tool resources come from Workflow defaults.