From 09b07751b6309bf796d15b2c5aaab4eb996f54cf Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 12 Aug 2026 17:29:30 +0800 Subject: [PATCH] feat: add configurable vision focus prompts --- backend/db/models.py | 1 + .../20260812_0019_add_vision_focus_prompt.py | 32 +++++ backend/models.py | 1 + backend/routes/assistants.py | 50 ++++++- backend/schemas.py | 10 ++ backend/services/config_resolver.py | 1 + backend/services/node_specs.py | 2 + backend/services/pipecat/pipeline.py | 24 ++-- backend/services/vision.py | 92 ++++++++++++- backend/services/workflow/agent.py | 8 +- backend/services/workflow/realtime.py | 11 +- backend/services/workflow_engine.py | 2 + backend/tests/test_vision_focus.py | 126 ++++++++++++++++++ .../assistant-editor/prompt-editor.tsx | 6 + .../src/components/assistant-editor/types.ts | 1 + .../editor/vision-config-section.tsx | 85 +++++++++++- .../src/components/pages/AssistantPage.tsx | 5 + .../components/pages/CreateAssistantPage.tsx | 1 + .../components/pages/TemplateLibraryPage.tsx | 1 + .../components/workflow/WorkflowCanvas.tsx | 1 + .../src/components/workflow/graph-settings.ts | 2 + .../workflow/panels/AgentNodePanel.tsx | 12 ++ .../workflow/panels/GlobalSettingsPanel.tsx | 10 ++ frontend/src/components/workflow/specs.ts | 3 + frontend/src/components/workflow/types.ts | 1 + frontend/src/lib/api.ts | 9 ++ 26 files changed, 477 insertions(+), 20 deletions(-) create mode 100644 backend/migrations/versions/20260812_0019_add_vision_focus_prompt.py create mode 100644 backend/tests/test_vision_focus.py diff --git a/backend/db/models.py b/backend/db/models.py index d832003..2e6baa8 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -160,6 +160,7 @@ class Assistant(Base): ForeignKey("model_resources.id", ondelete="SET NULL"), nullable=True, ) + vision_focus_prompt: Mapped[str] = mapped_column(Text, default="") # KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底 knowledge_base_id: Mapped[str | None] = mapped_column( diff --git a/backend/migrations/versions/20260812_0019_add_vision_focus_prompt.py b/backend/migrations/versions/20260812_0019_add_vision_focus_prompt.py new file mode 100644 index 0000000..cca2b0d --- /dev/null +++ b/backend/migrations/versions/20260812_0019_add_vision_focus_prompt.py @@ -0,0 +1,32 @@ +"""add assistant vision focus prompt + +Revision ID: 20260812_0019 +Revises: 20260810_0018 +""" + +from collections.abc import Sequence + +from alembic import op +import sqlalchemy as sa + + +revision: str = "20260812_0019" +down_revision: str | Sequence[str] | None = "20260810_0018" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "assistants", + sa.Column( + "vision_focus_prompt", + sa.Text(), + server_default=sa.text("''"), + nullable=False, + ), + ) + + +def downgrade() -> None: + op.drop_column("assistants", "vision_focus_prompt") diff --git a/backend/models.py b/backend/models.py index 07df225..f47a36e 100644 --- a/backend/models.py +++ b/backend/models.py @@ -92,6 +92,7 @@ class AssistantConfig(BaseModel): llm_support_image_input: bool = False vision_enabled: bool = False vision_model_resource_id: str | None = None + vision_focus_prompt: str = "" vision_model: str = "" vision_llm_interface_type: str = "openai-llm" vision_llm_values: dict = {} diff --git a/backend/routes/assistants.py b/backend/routes/assistants.py index a7ad45f..2e0ca67 100644 --- a/backend/routes/assistants.py +++ b/backend/routes/assistants.py @@ -2,7 +2,7 @@ import uuid -from models import RuntimeTool +from models import RuntimeModelResource, RuntimeTool from db.models import ( Assistant, AssistantModelBinding, @@ -13,12 +13,18 @@ from db.models import ( ) from db.session import get_session from fastapi import APIRouter, Depends, HTTPException -from schemas import AssistantOut, AssistantUpsert +from schemas import ( + AssistantOut, + AssistantUpsert, + VisionFocusPromptGenerateRequest, + VisionFocusPromptGenerateResult, +) from services.auth import require_admin from services.masking import mask, resolve_incoming_key from services.node_specs import graph_references, normalize_graph, validate_graph from services.system_tools import system_tool_kind from services.tool_policy import policy_for_tool +from services.vision import generate_vision_focus_prompt from services.workflow_engine import WorkflowEngine from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -73,6 +79,7 @@ def _validate_workflow(body: AssistantUpsert) -> None: # capability summary used by clients to decide whether to open the camera. body.vision_enabled = WorkflowEngine(body.graph).uses_vision() body.vision_model_resource_id = None + body.vision_focus_prompt = "" refs = graph_references(body.graph) body.tool_ids = list(dict.fromkeys([*body.tool_ids, *sorted(refs["tools"])])) @@ -389,6 +396,7 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut: startup=assistant.startup or {}, vision_enabled=assistant.vision_enabled, vision_model_resource_id=assistant.vision_model_resource_id, + vision_focus_prompt=assistant.vision_focus_prompt or "", model_resource_ids=await _resource_ids(session, assistant.id), knowledge_base_id=assistant.knowledge_base_id, knowledge_retrieval_config=assistant.knowledge_retrieval_config or {}, @@ -440,6 +448,43 @@ async def create_assistant( return await _to_out(session, assistant) +@router.post( + "/vision-focus/generate", + response_model=VisionFocusPromptGenerateResult, +) +async def generate_assistant_vision_focus( + body: VisionFocusPromptGenerateRequest, + session: AsyncSession = Depends(get_session), +): + """Generate an editable visual focus draft from an assistant prompt.""" + + resource = await session.get(ModelResource, body.model_resource_id) + if ( + resource is None + or resource.capability != "LLM" + or not resource.enabled + ): + raise HTTPException(400, "请选择可用的大语言模型生成视觉关注目标") + try: + focus_prompt = await generate_vision_focus_prompt( + RuntimeModelResource( + id=resource.id, + name=resource.name, + capability=resource.capability, + interface_type=resource.interface_type, + values=resource.values or {}, + secrets=resource.secrets or {}, + support_image_input=bool(resource.support_image_input), + ), + body.source_prompt, + ) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + except Exception as exc: + raise HTTPException(502, "生成视觉关注目标失败，请稍后重试") from exc + return VisionFocusPromptGenerateResult(focus_prompt=focus_prompt) + + @router.get("/{assistant_id}", response_model=AssistantOut) async def get_assistant( assistant_id: str, session: AsyncSession = Depends(get_session) @@ -468,6 +513,7 @@ async def duplicate_assistant( startup=dict(source.startup or {}), vision_enabled=source.vision_enabled, vision_model_resource_id=source.vision_model_resource_id, + vision_focus_prompt=source.vision_focus_prompt or "", knowledge_base_id=source.knowledge_base_id, knowledge_retrieval_config=dict(source.knowledge_retrieval_config or {}), prompt=source.prompt, diff --git a/backend/schemas.py b/backend/schemas.py index cc31a69..de8077b 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -198,6 +198,7 @@ class AssistantUpsert(CamelModel): startup: StartupConfig = Field(default_factory=StartupConfig) vision_enabled: bool = False vision_model_resource_id: str | None = None + vision_focus_prompt: str = Field(default="", max_length=4000) model_resource_ids: dict[ModelType, str] = Field(default_factory=dict) knowledge_base_id: str | None = None knowledge_retrieval_config: KnowledgeRetrievalConfig = Field( @@ -263,6 +264,15 @@ class AssistantOut(AssistantUpsert): updated_at: str | None = None +class VisionFocusPromptGenerateRequest(CamelModel): + source_prompt: str = Field(min_length=1, max_length=20000) + model_resource_id: str = Field(min_length=1, max_length=40) + + +class VisionFocusPromptGenerateResult(CamelModel): + focus_prompt: str + + class DynamicVariableDefinition(CamelModel): type: DynamicVariableType = "string" required: bool = False diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 9b64fc9..5028f2c 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -301,6 +301,7 @@ async def resolve_runtime_config( ), vision_enabled=assistant.vision_enabled, vision_model_resource_id=assistant.vision_model_resource_id, + vision_focus_prompt=assistant.vision_focus_prompt or "", vision_model=str(_value(vision_resource, "modelId", "")), vision_llm_interface_type=( vision_resource.interface_type if vision_resource else "openai-llm" diff --git a/backend/services/node_specs.py b/backend/services/node_specs.py index 3278c4c..1695690 100644 --- a/backend/services/node_specs.py +++ b/backend/services/node_specs.py @@ -207,6 +207,7 @@ def _normalize_agent_data(data: dict[str, Any]) -> None: data.get("ttsResourceId"), data.get("visionEnabled"), data.get("visionModelResourceId"), + data.get("visionFocusPrompt"), data.get("knowledgeBaseId"), data.get("toolIds"), ) @@ -267,6 +268,7 @@ def _normalize_settings(settings: dict[str, Any], *, global_prompt: str = "") -> settings.setdefault("defaultTtsResourceId", "") settings.setdefault("visionEnabled", False) settings.setdefault("visionModelResourceId", "") + settings.setdefault("visionFocusPrompt", "") settings.setdefault("toolIds", []) settings.setdefault("knowledgeBaseId", "") settings.setdefault("knowledgeMode", "automatic") diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index e5596f7..b4490be 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -31,7 +31,6 @@ from services.input_assets import consume_input_image from services.client_tools import ClientToolBroker from services.tool_policy import policy_for_tool from services.vision import ( - VISION_SYSTEM_HINT, VISION_TOOL_NAME, analyze_image_with_vision_model, config_with_main_llm_as_vision, @@ -39,6 +38,8 @@ from services.vision import ( image_data_uri, image_frame_from_jpeg, image_jpeg_bytes, + vision_system_hint, + vision_tool_description, ) from services.workflow_engine import WorkflowEngine @@ -348,7 +349,7 @@ async def run_pipeline( def with_vision_hint(text: str) -> str: hints = [] if vision_enabled and cfg.type != "workflow": - hints.append(VISION_SYSTEM_HINT) + hints.append(vision_system_hint(cfg.vision_focus_prompt)) if cfg.knowledge_base_id: hints.append( AUTOMATIC_KNOWLEDGE_SYSTEM_HINT @@ -449,6 +450,7 @@ async def run_pipeline( workflow_vision_scope: dict[str, Any] = { "enabled": False, "vision_model_resource_id": None, + "vision_focus_prompt": "", "llm_resource_id": None, } @@ -458,10 +460,7 @@ async def run_pipeline( return vision_native_mode vision_schema = FunctionSchema( name=VISION_TOOL_NAME, - description=( - "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" - "人/物品/环境状态或需要视觉判断时调用。" - ), + description=vision_tool_description(cfg.vision_focus_prompt), properties={ "question": { "type": "string", @@ -572,10 +571,15 @@ async def run_pipeline( resource_id = vision_resource_id or llm_resource_id resource = cfg.workflow_model_resources.get(resource_id) if resource: - return config_with_vision_resource(cfg, resource) - if not vision_resource_id: - return config_with_main_llm_as_vision(cfg) - raise ValueError(f"视觉模型资源未加载:{vision_resource_id}") + result = config_with_vision_resource(cfg, resource) + elif not vision_resource_id: + result = config_with_main_llm_as_vision(cfg) + else: + raise ValueError(f"视觉模型资源未加载:{vision_resource_id}") + result.vision_focus_prompt = str( + workflow_vision_scope.get("vision_focus_prompt") or "" + ).strip() + return result if cfg.type == "workflow" and vision_enabled: async def native_flow_fetch_user_image(params: FunctionCallParams) -> None: diff --git a/backend/services/vision.py b/backend/services/vision.py index a819ca3..0a56224 100644 --- a/backend/services/vision.py +++ b/backend/services/vision.py @@ -22,6 +22,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断，请明确说明不确定。" ) +VISION_FOCUS_GENERATION_SYSTEM_PROMPT = ( + "你负责把对话助手提示词整理成一段可编辑的视觉关注目标。" + "只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；" + "不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、" + "意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。" +) def _require(value: str, label: str) -> str: @@ -30,6 +36,85 @@ def _require(value: str, label: str) -> str: raise ValueError(f"缺少模型资源配置: {label}") +def vision_system_hint(focus_prompt: str = "") -> str: + """Describe when to use vision and keep a configured business focus visible.""" + + focus = focus_prompt.strip() + if not focus: + return VISION_SYSTEM_HINT + return ( + f"{VISION_SYSTEM_HINT}\n" + f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}" + ) + + +def vision_analysis_system_prompt(focus_prompt: str = "") -> str: + """Keep the safety baseline fixed while appending an editable business focus.""" + + focus = focus_prompt.strip() + if not focus: + return VISION_ANALYSIS_SYSTEM_PROMPT + return ( + f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n" + f"[业务视觉关注目标]\n{focus}\n" + "该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。" + ) + + +def vision_tool_description(focus_prompt: str = "") -> str: + base = ( + "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" + "人/物品/环境状态或需要视觉判断时调用。" + ) + focus = " ".join(focus_prompt.split()) + return f"{base} 当前业务重点：{focus[:600]}" if focus else base + + +async def generate_vision_focus_prompt( + resource: RuntimeModelResource, + source_prompt: str, +) -> str: + """Generate one editable focus draft with an existing LLM resource.""" + + if resource.interface_type not in {"openai-llm", "dashscope-llm"}: + raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}") + source = source_prompt.strip() + if not source: + raise ValueError("助手提示词不能为空") + values = resource.values or {} + secrets = resource.secrets or {} + extra_body = values.get("extraBody") + extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} + client = AsyncOpenAI( + api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"), + base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"), + ) + try: + response = await client.chat.completions.create( + model=_require(str(values.get("modelId") or ""), "LLM modelId"), + messages=[ + {"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT}, + {"role": "user", "content": source}, + ], + **extra, + ) + finally: + await client.close() + content = response.choices[0].message.content if response.choices else "" + result = ( + content.strip() + if isinstance(content, str) + else str(content or "").strip() + ) + if not result: + raise ValueError("模型没有生成有效的视觉关注目标") + if result.startswith("```") and result.endswith("```"): + result = result[3:-3].strip() + if result.startswith("text"): + result = result[4:].strip() + return result[:4000] + + def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes: """Encode one Pipecat camera frame as a storage- and model-ready JPEG.""" if not frame.format: @@ -86,7 +171,12 @@ async def analyze_image_with_vision_model( response = await client.chat.completions.create( model=_require(cfg.vision_model, "Vision LLM modelId"), messages=[ - {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, + { + "role": "system", + "content": vision_analysis_system_prompt( + cfg.vision_focus_prompt + ), + }, { "role": "user", "content": [ diff --git a/backend/services/workflow/agent.py b/backend/services/workflow/agent.py index ff1dd17..b12358d 100644 --- a/backend/services/workflow/agent.py +++ b/backend/services/workflow/agent.py @@ -11,7 +11,7 @@ from pipecat.services.settings import LLMSettings from services.brains.base import BrainRuntime from services.runtime_variables import DynamicVariableStore -from services.vision import VISION_SYSTEM_HINT +from services.vision import vision_system_hint from services.workflow_engine import WorkflowEngine from services.workflow_router import WorkflowLLMRouter @@ -47,7 +47,10 @@ class WorkflowAgentStage: stage_prompt = self._engine.prompt_for(node_id, self._store) stage = self._engine.agent_stage_config(node_id) if stage.vision_enabled: - stage_prompt = f"{stage_prompt}\n\n[视觉能力]\n{VISION_SYSTEM_HINT}" + stage_prompt = ( + f"{stage_prompt}\n\n[视觉能力]\n" + f"{vision_system_hint(stage.vision_focus_prompt)}" + ) instruction = ( EDGE_TOOL_STAGE_INSTRUCTION if self._engine.llm_routing_mode() == "edge_tool" @@ -109,6 +112,7 @@ class WorkflowAgentStage: { "enabled": stage.vision_enabled, "vision_model_resource_id": stage.vision_model_resource_id, + "vision_focus_prompt": stage.vision_focus_prompt, "llm_resource_id": stage.llm_resource_id, } ) diff --git a/backend/services/workflow/realtime.py b/backend/services/workflow/realtime.py index fe50051..78944de 100644 --- a/backend/services/workflow/realtime.py +++ b/backend/services/workflow/realtime.py @@ -31,10 +31,11 @@ from services.system_tools import state_update_properties, system_tool_kind from services.tool_executor import ToolExecutionError, ToolExecutor from services.tool_policy import policy_for_tool from services.vision import ( - VISION_SYSTEM_HINT, VISION_TOOL_NAME, analyze_image_with_vision_model, config_with_vision_resource, + vision_system_hint, + vision_tool_description, ) from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION from services.workflow.models import WorkflowRuntimeState, WorkflowStatus @@ -407,12 +408,13 @@ class WorkflowRealtimeController: return RealtimeActivation(continue_response=generate) def _agent_prompt(self, node_id: str) -> str: + stage = self._engine.agent_stage_config(node_id) prompt = ( f"{self._engine.prompt_for(node_id, self._store)}\n\n" f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}" ) - if self._engine.agent_stage_config(node_id).vision_enabled: - prompt = f"{prompt}\n{VISION_SYSTEM_HINT}" + if stage.vision_enabled: + prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}" return prompt def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]: @@ -603,6 +605,7 @@ class WorkflowRealtimeController: try: if vision_cfg is not None: + vision_cfg.vision_focus_prompt = stage.vision_focus_prompt observation = await analyze_image_with_vision_model( vision_cfg, frame, @@ -634,7 +637,7 @@ class WorkflowRealtimeController: return ( RealtimeTool( name=VISION_TOOL_NAME, - description="获取用户摄像头的当前画面，并回答一个视觉问题。", + description=vision_tool_description(stage.vision_focus_prompt), properties={ "question": { "type": "string", diff --git a/backend/services/workflow_engine.py b/backend/services/workflow_engine.py index 3672117..8e12316 100644 --- a/backend/services/workflow_engine.py +++ b/backend/services/workflow_engine.py @@ -23,6 +23,7 @@ class AgentStageConfig: tts_resource_id: str vision_enabled: bool vision_model_resource_id: str | None + vision_focus_prompt: str tool_ids: tuple[str, ...] state_variable_names: tuple[str, ...] knowledge_base_id: str | None @@ -151,6 +152,7 @@ class WorkflowEngine: vision_model_resource_id=( str(source.get("visionModelResourceId") or "") or None ), + vision_focus_prompt=str(source.get("visionFocusPrompt") or "").strip(), tool_ids=tuple(str(tool_id) for tool_id in source.get("toolIds") or []), # Variable write scope always belongs to the Agent node, even when # its selected tool resources come from Workflow defaults. diff --git a/backend/tests/test_vision_focus.py b/backend/tests/test_vision_focus.py new file mode 100644 index 0000000..fdb5a4c --- /dev/null +++ b/backend/tests/test_vision_focus.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from models import RuntimeModelResource +from services.vision import ( + VISION_ANALYSIS_SYSTEM_PROMPT, + VISION_SYSTEM_HINT, + generate_vision_focus_prompt, + vision_analysis_system_prompt, + vision_system_hint, + vision_tool_description, +) +from services.workflow_engine import WorkflowEngine + + +class VisionFocusPromptTest(unittest.TestCase): + def test_focus_extends_but_does_not_replace_safety_prompts(self): + focus = "优先识别车辆数量和人员状态，只报告可确认的信息。" + + system_hint = vision_system_hint(focus) + analysis_prompt = vision_analysis_system_prompt(focus) + + self.assertIn(VISION_SYSTEM_HINT, system_hint) + self.assertIn(focus, system_hint) + self.assertIn(VISION_ANALYSIS_SYSTEM_PROMPT, analysis_prompt) + self.assertIn(focus, analysis_prompt) + self.assertIn("优先回答当前用户问题", analysis_prompt) + + def test_empty_focus_preserves_previous_runtime_prompts(self): + self.assertEqual(vision_system_hint(), VISION_SYSTEM_HINT) + self.assertEqual( + vision_analysis_system_prompt(), + VISION_ANALYSIS_SYSTEM_PROMPT, + ) + + def test_tool_description_compacts_and_limits_focus(self): + description = vision_tool_description("车辆\n 数量 " * 300) + + self.assertIn("当前业务重点", description) + self.assertNotIn("\n", description) + self.assertLess(len(description), 700) + + def test_workflow_resolves_global_and_node_focus(self): + graph = { + "specVersion": 3, + "settings": { + "visionEnabled": True, + "visionFocusPrompt": "全局关注目标", + }, + "nodes": [ + { + "id": "global-agent", + "type": "agent", + "data": {"name": "全局 Agent", "prompt": "处理任务"}, + }, + { + "id": "custom-agent", + "type": "agent", + "data": { + "name": "独立 Agent", + "prompt": "检查车辆", + "inheritGlobalConfig": False, + "visionEnabled": True, + "visionFocusPrompt": "节点关注目标", + }, + }, + ], + "edges": [], + } + engine = WorkflowEngine(graph) + + self.assertEqual( + engine.agent_stage_config("global-agent").vision_focus_prompt, + "全局关注目标", + ) + self.assertEqual( + engine.agent_stage_config("custom-agent").vision_focus_prompt, + "节点关注目标", + ) + + +class VisionFocusGenerationTest(unittest.IsolatedAsyncioTestCase): + async def test_generates_editable_focus_with_selected_resource(self): + resource = RuntimeModelResource( + id="llm_1", + name="生成模型", + capability="LLM", + interface_type="openai-llm", + values={ + "apiUrl": "https://llm.example/v1", + "modelId": "focus-model", + }, + secrets={"apiKey": "secret"}, + ) + client = MagicMock() + client.chat.completions.create = AsyncMock( + return_value=SimpleNamespace( + choices=[ + SimpleNamespace( + message=SimpleNamespace( + content="优先识别车辆数量、损伤位置和人员状态。" + ) + ) + ] + ) + ) + client.close = AsyncMock() + + with patch("services.vision.AsyncOpenAI", return_value=client): + result = await generate_vision_focus_prompt( + resource, + "你是交通事故处理助手，需要了解车辆和人员情况。", + ) + + self.assertEqual(result, "优先识别车辆数量、损伤位置和人员状态。") + request = client.chat.completions.create.await_args.kwargs + self.assertEqual(request["model"], "focus-model") + self.assertIn("交通事故处理助手", request["messages"][1]["content"]) + client.close.assert_awaited_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/frontend/src/components/assistant-editor/prompt-editor.tsx b/frontend/src/components/assistant-editor/prompt-editor.tsx index f04f5f5..b6bfb5c 100644 --- a/frontend/src/components/assistant-editor/prompt-editor.tsx +++ b/frontend/src/components/assistant-editor/prompt-editor.tsx @@ -648,12 +648,18 @@ export function PromptEditor({ hint="开启后，助手会获得读取当前视频画面的工具。选择「模型自己」时，大语言模型必须支持图片输入。" enabled={form.visionEnabled} modelResourceId={form.visionModelResourceId} + focusPrompt={form.visionFocusPrompt} + sourcePrompt={form.prompt} + generatorModelResourceId={form.model} mainModelResourceId={form.model} modelOptions={visionModelOptions} onEnabledChange={handlePromptVisionEnabledChange} onModelResourceIdChange={(value) => updateForm("visionModelResourceId", value) } + onFocusPromptChange={(value) => + updateForm("visionFocusPrompt", value) + } /> )} diff --git a/frontend/src/components/assistant-editor/types.ts b/frontend/src/components/assistant-editor/types.ts index d4f8b7a..ae9084e 100644 --- a/frontend/src/components/assistant-editor/types.ts +++ b/frontend/src/components/assistant-editor/types.ts @@ -26,5 +26,6 @@ export type AssistantForm = { startup: StartupConfig; visionEnabled: boolean; visionModelResourceId: string; + visionFocusPrompt: string; toolIds: string[]; }; diff --git a/frontend/src/components/editor/vision-config-section.tsx b/frontend/src/components/editor/vision-config-section.tsx index 40ae4c0..6ddd66a 100644 --- a/frontend/src/components/editor/vision-config-section.tsx +++ b/frontend/src/components/editor/vision-config-section.tsx @@ -1,12 +1,16 @@ "use client"; -import { Camera } from "lucide-react"; +import { useState } from "react"; +import { Camera, Loader2, Sparkles } from "lucide-react"; import { ResourceSelectField, ToggleRow, } from "@/components/assistant-editor/editor-controls"; import { SectionCard } from "@/components/editor/section-card"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; +import { assistantsApi } from "@/lib/api"; type VisionModelOption = { value: string; @@ -18,22 +22,32 @@ export function VisionConfigSection({ hint, enabled, modelResourceId, + focusPrompt, + sourcePrompt, + generatorModelResourceId, mainModelResourceId, mainModelSupportsImageInput, modelOptions, onEnabledChange, onModelResourceIdChange, + onFocusPromptChange, }: { description: string; hint: string; enabled: boolean; modelResourceId: string; + focusPrompt: string; + sourcePrompt: string; + generatorModelResourceId: string; mainModelResourceId: string; mainModelSupportsImageInput?: boolean; modelOptions: VisionModelOption[]; onEnabledChange: (enabled: boolean) => void; onModelResourceIdChange: (modelResourceId: string) => void; + onFocusPromptChange: (focusPrompt: string) => void; }) { + const [generating, setGenerating] = useState(false); + const [generationError, setGenerationError] = useState(""); const mainModelSupportsVision = mainModelSupportsImageInput ?? modelOptions.some((option) => option.value === mainModelResourceId); @@ -41,6 +55,25 @@ export function VisionConfigSection({ (option) => option.value !== mainModelResourceId, ); + async function generateFocusPrompt() { + if (!sourcePrompt.trim() || !generatorModelResourceId || generating) return; + setGenerating(true); + setGenerationError(""); + try { + const result = await assistantsApi.generateVisionFocus({ + sourcePrompt, + modelResourceId: generatorModelResourceId, + }); + onFocusPromptChange(result.focusPrompt); + } catch (error) { + setGenerationError( + error instanceof Error ? error.message : "生成视觉关注目标失败", + ); + } finally { + setGenerating(false); + } + } + return ( } @@ -67,6 +100,56 @@ export function VisionConfigSection({ 当前模型未标记支持图片输入，请选择独立视觉模型。

)} +
+
+ + +
+