feat: add configurable vision focus prompts

backend/db/models.py

@@ -160,6 +160,7 @@ class Assistant(Base):
         ForeignKey("model_resources.id", ondelete="SET NULL"),
         nullable=True,
     )
+    vision_focus_prompt: Mapped[str] = mapped_column(Text, default="")
 
     # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
     knowledge_base_id: Mapped[str | None] = mapped_column(

backend/migrations/versions/20260812_0019_add_vision_focus_prompt.py

@@ -0,0 +1,32 @@
+"""add assistant vision focus prompt
+
+Revision ID: 20260812_0019
+Revises: 20260810_0018
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision: str = "20260812_0019"
+down_revision: str | Sequence[str] | None = "20260810_0018"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.add_column(
+        "assistants",
+        sa.Column(
+            "vision_focus_prompt",
+            sa.Text(),
+            server_default=sa.text("''"),
+            nullable=False,
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_column("assistants", "vision_focus_prompt")

backend/models.py

@@ -92,6 +92,7 @@ class AssistantConfig(BaseModel):
     llm_support_image_input: bool = False
     vision_enabled: bool = False
     vision_model_resource_id: str | None = None
+    vision_focus_prompt: str = ""
     vision_model: str = ""
     vision_llm_interface_type: str = "openai-llm"
     vision_llm_values: dict = {}

backend/routes/assistants.py

@@ -2,7 +2,7 @@
 
 import uuid
 
-from models import RuntimeTool
+from models import RuntimeModelResource, RuntimeTool
 from db.models import (
     Assistant,
     AssistantModelBinding,
@@ -13,12 +13,18 @@ from db.models import (
 )
 from db.session import get_session
 from fastapi import APIRouter, Depends, HTTPException
-from schemas import AssistantOut, AssistantUpsert
+from schemas import (
+    AssistantOut,
+    AssistantUpsert,
+    VisionFocusPromptGenerateRequest,
+    VisionFocusPromptGenerateResult,
+)
 from services.auth import require_admin
 from services.masking import mask, resolve_incoming_key
 from services.node_specs import graph_references, normalize_graph, validate_graph
 from services.system_tools import system_tool_kind
 from services.tool_policy import policy_for_tool
+from services.vision import generate_vision_focus_prompt
 from services.workflow_engine import WorkflowEngine
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
@@ -73,6 +79,7 @@ def _validate_workflow(body: AssistantUpsert) -> None:
     # capability summary used by clients to decide whether to open the camera.
     body.vision_enabled = WorkflowEngine(body.graph).uses_vision()
     body.vision_model_resource_id = None
+    body.vision_focus_prompt = ""
     refs = graph_references(body.graph)
     body.tool_ids = list(dict.fromkeys([*body.tool_ids, *sorted(refs["tools"])]))
 
@@ -389,6 +396,7 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
         startup=assistant.startup or {},
         vision_enabled=assistant.vision_enabled,
         vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_focus_prompt=assistant.vision_focus_prompt or "",
         model_resource_ids=await _resource_ids(session, assistant.id),
         knowledge_base_id=assistant.knowledge_base_id,
         knowledge_retrieval_config=assistant.knowledge_retrieval_config or {},
@@ -440,6 +448,43 @@ async def create_assistant(
     return await _to_out(session, assistant)
 
 
+@router.post(
+    "/vision-focus/generate",
+    response_model=VisionFocusPromptGenerateResult,
+)
+async def generate_assistant_vision_focus(
+    body: VisionFocusPromptGenerateRequest,
+    session: AsyncSession = Depends(get_session),
+):
+    """Generate an editable visual focus draft from an assistant prompt."""
+
+    resource = await session.get(ModelResource, body.model_resource_id)
+    if (
+        resource is None
+        or resource.capability != "LLM"
+        or not resource.enabled
+    ):
+        raise HTTPException(400, "请选择可用的大语言模型生成视觉关注目标")
+    try:
+        focus_prompt = await generate_vision_focus_prompt(
+            RuntimeModelResource(
+                id=resource.id,
+                name=resource.name,
+                capability=resource.capability,
+                interface_type=resource.interface_type,
+                values=resource.values or {},
+                secrets=resource.secrets or {},
+                support_image_input=bool(resource.support_image_input),
+            ),
+            body.source_prompt,
+        )
+    except ValueError as exc:
+        raise HTTPException(400, str(exc)) from exc
+    except Exception as exc:
+        raise HTTPException(502, "生成视觉关注目标失败，请稍后重试") from exc
+    return VisionFocusPromptGenerateResult(focus_prompt=focus_prompt)
+
+
 @router.get("/{assistant_id}", response_model=AssistantOut)
 async def get_assistant(
     assistant_id: str, session: AsyncSession = Depends(get_session)
@@ -468,6 +513,7 @@ async def duplicate_assistant(
         startup=dict(source.startup or {}),
         vision_enabled=source.vision_enabled,
         vision_model_resource_id=source.vision_model_resource_id,
+        vision_focus_prompt=source.vision_focus_prompt or "",
         knowledge_base_id=source.knowledge_base_id,
         knowledge_retrieval_config=dict(source.knowledge_retrieval_config or {}),
         prompt=source.prompt,

backend/schemas.py

@@ -198,6 +198,7 @@ class AssistantUpsert(CamelModel):
     startup: StartupConfig = Field(default_factory=StartupConfig)
     vision_enabled: bool = False
     vision_model_resource_id: str | None = None
+    vision_focus_prompt: str = Field(default="", max_length=4000)
     model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
     knowledge_base_id: str | None = None
     knowledge_retrieval_config: KnowledgeRetrievalConfig = Field(
@@ -263,6 +264,15 @@ class AssistantOut(AssistantUpsert):
     updated_at: str | None = None
 
 
+class VisionFocusPromptGenerateRequest(CamelModel):
+    source_prompt: str = Field(min_length=1, max_length=20000)
+    model_resource_id: str = Field(min_length=1, max_length=40)
+
+
+class VisionFocusPromptGenerateResult(CamelModel):
+    focus_prompt: str
+
+
 class DynamicVariableDefinition(CamelModel):
     type: DynamicVariableType = "string"
     required: bool = False

backend/services/config_resolver.py

@@ -301,6 +301,7 @@ async def resolve_runtime_config(
         ),
         vision_enabled=assistant.vision_enabled,
         vision_model_resource_id=assistant.vision_model_resource_id,
+        vision_focus_prompt=assistant.vision_focus_prompt or "",
         vision_model=str(_value(vision_resource, "modelId", "")),
         vision_llm_interface_type=(
             vision_resource.interface_type if vision_resource else "openai-llm"

backend/services/node_specs.py

@@ -207,6 +207,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None:
                 data.get("ttsResourceId"),
                 data.get("visionEnabled"),
                 data.get("visionModelResourceId"),
+                data.get("visionFocusPrompt"),
                 data.get("knowledgeBaseId"),
                 data.get("toolIds"),
             )
@@ -267,6 +268,7 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") ->
     settings.setdefault("defaultTtsResourceId", "")
     settings.setdefault("visionEnabled", False)
     settings.setdefault("visionModelResourceId", "")
+    settings.setdefault("visionFocusPrompt", "")
     settings.setdefault("toolIds", [])
     settings.setdefault("knowledgeBaseId", "")
     settings.setdefault("knowledgeMode", "automatic")

backend/services/pipecat/pipeline.py

@@ -31,7 +31,6 @@ from services.input_assets import consume_input_image
 from services.client_tools import ClientToolBroker
 from services.tool_policy import policy_for_tool
 from services.vision import (
-    VISION_SYSTEM_HINT,
     VISION_TOOL_NAME,
     analyze_image_with_vision_model,
     config_with_main_llm_as_vision,
@@ -39,6 +38,8 @@ from services.vision import (
     image_data_uri,
     image_frame_from_jpeg,
     image_jpeg_bytes,
+    vision_system_hint,
+    vision_tool_description,
 )
 from services.workflow_engine import WorkflowEngine
 
@@ -348,7 +349,7 @@ async def run_pipeline(
     def with_vision_hint(text: str) -> str:
         hints = []
         if vision_enabled and cfg.type != "workflow":
-            hints.append(VISION_SYSTEM_HINT)
+            hints.append(vision_system_hint(cfg.vision_focus_prompt))
         if cfg.knowledge_base_id:
             hints.append(
                 AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
@@ -449,6 +450,7 @@ async def run_pipeline(
     workflow_vision_scope: dict[str, Any] = {
         "enabled": False,
         "vision_model_resource_id": None,
+        "vision_focus_prompt": "",
         "llm_resource_id": None,
     }
 
@@ -458,10 +460,7 @@ async def run_pipeline(
         return vision_native_mode
     vision_schema = FunctionSchema(
         name=VISION_TOOL_NAME,
-        description=(
-            "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
-            "人/物品/环境状态或需要视觉判断时调用。"
-        ),
+        description=vision_tool_description(cfg.vision_focus_prompt),
         properties={
             "question": {
                 "type": "string",
@@ -572,10 +571,15 @@ async def run_pipeline(
         resource_id = vision_resource_id or llm_resource_id
         resource = cfg.workflow_model_resources.get(resource_id)
         if resource:
-            return config_with_vision_resource(cfg, resource)
-        if not vision_resource_id:
-            return config_with_main_llm_as_vision(cfg)
-        raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
+            result = config_with_vision_resource(cfg, resource)
+        elif not vision_resource_id:
+            result = config_with_main_llm_as_vision(cfg)
+        else:
+            raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
+        result.vision_focus_prompt = str(
+            workflow_vision_scope.get("vision_focus_prompt") or ""
+        ).strip()
+        return result
 
     if cfg.type == "workflow" and vision_enabled:
         async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:

backend/services/vision.py

@@ -22,6 +22,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
     "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
     "如果画面不足以判断，请明确说明不确定。"
 )
+VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
+    "你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
+    "只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
+    "不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
+    "意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
+)
 
 
 def _require(value: str, label: str) -> str:
@@ -30,6 +36,85 @@ def _require(value: str, label: str) -> str:
     raise ValueError(f"缺少模型资源配置: {label}")
 
 
+def vision_system_hint(focus_prompt: str = "") -> str:
+    """Describe when to use vision and keep a configured business focus visible."""
+
+    focus = focus_prompt.strip()
+    if not focus:
+        return VISION_SYSTEM_HINT
+    return (
+        f"{VISION_SYSTEM_HINT}\n"
+        f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
+    )
+
+
+def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
+    """Keep the safety baseline fixed while appending an editable business focus."""
+
+    focus = focus_prompt.strip()
+    if not focus:
+        return VISION_ANALYSIS_SYSTEM_PROMPT
+    return (
+        f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
+        f"[业务视觉关注目标]\n{focus}\n"
+        "该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
+    )
+
+
+def vision_tool_description(focus_prompt: str = "") -> str:
+    base = (
+        "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
+        "人/物品/环境状态或需要视觉判断时调用。"
+    )
+    focus = " ".join(focus_prompt.split())
+    return f"{base} 当前业务重点：{focus[:600]}" if focus else base
+
+
+async def generate_vision_focus_prompt(
+    resource: RuntimeModelResource,
+    source_prompt: str,
+) -> str:
+    """Generate one editable focus draft with an existing LLM resource."""
+
+    if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
+        raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
+    source = source_prompt.strip()
+    if not source:
+        raise ValueError("助手提示词不能为空")
+    values = resource.values or {}
+    secrets = resource.secrets or {}
+    extra_body = values.get("extraBody")
+    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
+    client = AsyncOpenAI(
+        api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
+        base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
+    )
+    try:
+        response = await client.chat.completions.create(
+            model=_require(str(values.get("modelId") or ""), "LLM modelId"),
+            messages=[
+                {"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
+                {"role": "user", "content": source},
+            ],
+            **extra,
+        )
+    finally:
+        await client.close()
+    content = response.choices[0].message.content if response.choices else ""
+    result = (
+        content.strip()
+        if isinstance(content, str)
+        else str(content or "").strip()
+    )
+    if not result:
+        raise ValueError("模型没有生成有效的视觉关注目标")
+    if result.startswith("```") and result.endswith("```"):
+        result = result[3:-3].strip()
+        if result.startswith("text"):
+            result = result[4:].strip()
+    return result[:4000]
+
+
 def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
     """Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
     if not frame.format:
@@ -86,7 +171,12 @@ async def analyze_image_with_vision_model(
         response = await client.chat.completions.create(
             model=_require(cfg.vision_model, "Vision LLM modelId"),
             messages=[
-                {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
+                {
+                    "role": "system",
+                    "content": vision_analysis_system_prompt(
+                        cfg.vision_focus_prompt
+                    ),
+                },
                 {
                     "role": "user",
                     "content": [

backend/services/workflow/agent.py

@@ -11,7 +11,7 @@ from pipecat.services.settings import LLMSettings
 
 from services.brains.base import BrainRuntime
 from services.runtime_variables import DynamicVariableStore
-from services.vision import VISION_SYSTEM_HINT
+from services.vision import vision_system_hint
 from services.workflow_engine import WorkflowEngine
 from services.workflow_router import WorkflowLLMRouter
 
@@ -47,7 +47,10 @@ class WorkflowAgentStage:
         stage_prompt = self._engine.prompt_for(node_id, self._store)
         stage = self._engine.agent_stage_config(node_id)
         if stage.vision_enabled:
-            stage_prompt = f"{stage_prompt}\n\n[视觉能力]\n{VISION_SYSTEM_HINT}"
+            stage_prompt = (
+                f"{stage_prompt}\n\n[视觉能力]\n"
+                f"{vision_system_hint(stage.vision_focus_prompt)}"
+            )
         instruction = (
             EDGE_TOOL_STAGE_INSTRUCTION
             if self._engine.llm_routing_mode() == "edge_tool"
@@ -109,6 +112,7 @@ class WorkflowAgentStage:
                 {
                     "enabled": stage.vision_enabled,
                     "vision_model_resource_id": stage.vision_model_resource_id,
+                    "vision_focus_prompt": stage.vision_focus_prompt,
                     "llm_resource_id": stage.llm_resource_id,
                 }
             )

backend/services/workflow/realtime.py

@@ -31,10 +31,11 @@ from services.system_tools import state_update_properties, system_tool_kind
 from services.tool_executor import ToolExecutionError, ToolExecutor
 from services.tool_policy import policy_for_tool
 from services.vision import (
-    VISION_SYSTEM_HINT,
     VISION_TOOL_NAME,
     analyze_image_with_vision_model,
     config_with_vision_resource,
+    vision_system_hint,
+    vision_tool_description,
 )
 from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
 from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
@@ -407,12 +408,13 @@ class WorkflowRealtimeController:
         return RealtimeActivation(continue_response=generate)
 
     def _agent_prompt(self, node_id: str) -> str:
+        stage = self._engine.agent_stage_config(node_id)
         prompt = (
             f"{self._engine.prompt_for(node_id, self._store)}\n\n"
             f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
         )
-        if self._engine.agent_stage_config(node_id).vision_enabled:
-            prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
+        if stage.vision_enabled:
+            prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
         return prompt
 
     def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
@@ -603,6 +605,7 @@ class WorkflowRealtimeController:
 
             try:
                 if vision_cfg is not None:
+                    vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
                     observation = await analyze_image_with_vision_model(
                         vision_cfg,
                         frame,
@@ -634,7 +637,7 @@ class WorkflowRealtimeController:
         return (
             RealtimeTool(
                 name=VISION_TOOL_NAME,
-                description="获取用户摄像头的当前画面，并回答一个视觉问题。",
+                description=vision_tool_description(stage.vision_focus_prompt),
                 properties={
                     "question": {
                         "type": "string",

backend/services/workflow_engine.py

@@ -23,6 +23,7 @@ class AgentStageConfig:
     tts_resource_id: str
     vision_enabled: bool
     vision_model_resource_id: str | None
+    vision_focus_prompt: str
     tool_ids: tuple[str, ...]
     state_variable_names: tuple[str, ...]
     knowledge_base_id: str | None
@@ -151,6 +152,7 @@ class WorkflowEngine:
             vision_model_resource_id=(
                 str(source.get("visionModelResourceId") or "") or None
             ),
+            vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(),
             tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []),
             # Variable write scope always belongs to the Agent node, even when
             # its selected tool resources come from Workflow defaults.

backend/tests/test_vision_focus.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from models import RuntimeModelResource
+from services.vision import (
+    VISION_ANALYSIS_SYSTEM_PROMPT,
+    VISION_SYSTEM_HINT,
+    generate_vision_focus_prompt,
+    vision_analysis_system_prompt,
+    vision_system_hint,
+    vision_tool_description,
+)
+from services.workflow_engine import WorkflowEngine
+
+
+class VisionFocusPromptTest(unittest.TestCase):
+    def test_focus_extends_but_does_not_replace_safety_prompts(self):
+        focus = "优先识别车辆数量和人员状态，只报告可确认的信息。"
+
+        system_hint = vision_system_hint(focus)
+        analysis_prompt = vision_analysis_system_prompt(focus)
+
+        self.assertIn(VISION_SYSTEM_HINT, system_hint)
+        self.assertIn(focus, system_hint)
+        self.assertIn(VISION_ANALYSIS_SYSTEM_PROMPT, analysis_prompt)
+        self.assertIn(focus, analysis_prompt)
+        self.assertIn("优先回答当前用户问题", analysis_prompt)
+
+    def test_empty_focus_preserves_previous_runtime_prompts(self):
+        self.assertEqual(vision_system_hint(), VISION_SYSTEM_HINT)
+        self.assertEqual(
+            vision_analysis_system_prompt(),
+            VISION_ANALYSIS_SYSTEM_PROMPT,
+        )
+
+    def test_tool_description_compacts_and_limits_focus(self):
+        description = vision_tool_description("车辆\n  数量 " * 300)
+
+        self.assertIn("当前业务重点", description)
+        self.assertNotIn("\n", description)
+        self.assertLess(len(description), 700)
+
+    def test_workflow_resolves_global_and_node_focus(self):
+        graph = {
+            "specVersion": 3,
+            "settings": {
+                "visionEnabled": True,
+                "visionFocusPrompt": "全局关注目标",
+            },
+            "nodes": [
+                {
+                    "id": "global-agent",
+                    "type": "agent",
+                    "data": {"name": "全局 Agent", "prompt": "处理任务"},
+                },
+                {
+                    "id": "custom-agent",
+                    "type": "agent",
+                    "data": {
+                        "name": "独立 Agent",
+                        "prompt": "检查车辆",
+                        "inheritGlobalConfig": False,
+                        "visionEnabled": True,
+                        "visionFocusPrompt": "节点关注目标",
+                    },
+                },
+            ],
+            "edges": [],
+        }
+        engine = WorkflowEngine(graph)
+
+        self.assertEqual(
+            engine.agent_stage_config("global-agent").vision_focus_prompt,
+            "全局关注目标",
+        )
+        self.assertEqual(
+            engine.agent_stage_config("custom-agent").vision_focus_prompt,
+            "节点关注目标",
+        )
+
+
+class VisionFocusGenerationTest(unittest.IsolatedAsyncioTestCase):
+    async def test_generates_editable_focus_with_selected_resource(self):
+        resource = RuntimeModelResource(
+            id="llm_1",
+            name="生成模型",
+            capability="LLM",
+            interface_type="openai-llm",
+            values={
+                "apiUrl": "https://llm.example/v1",
+                "modelId": "focus-model",
+            },
+            secrets={"apiKey": "secret"},
+        )
+        client = MagicMock()
+        client.chat.completions.create = AsyncMock(
+            return_value=SimpleNamespace(
+                choices=[
+                    SimpleNamespace(
+                        message=SimpleNamespace(
+                            content="优先识别车辆数量、损伤位置和人员状态。"
+                        )
+                    )
+                ]
+            )
+        )
+        client.close = AsyncMock()
+
+        with patch("services.vision.AsyncOpenAI", return_value=client):
+            result = await generate_vision_focus_prompt(
+                resource,
+                "你是交通事故处理助手，需要了解车辆和人员情况。",
+            )
+
+        self.assertEqual(result, "优先识别车辆数量、损伤位置和人员状态。")
+        request = client.chat.completions.create.await_args.kwargs
+        self.assertEqual(request["model"], "focus-model")
+        self.assertIn("交通事故处理助手", request["messages"][1]["content"])
+        client.close.assert_awaited_once()
+
+
+if __name__ == "__main__":
+    unittest.main()