feat: add configurable vision focus prompts

This commit is contained in:
Xin Wang
2026-08-12 17:29:30 +08:00
parent 7560f9a525
commit 09b07751b6
26 changed files with 477 additions and 20 deletions
+7 -4
View File
@@ -31,10 +31,11 @@ from services.system_tools import state_update_properties, system_tool_kind
from services.tool_executor import ToolExecutionError, ToolExecutor
from services.tool_policy import policy_for_tool
from services.vision import (
VISION_SYSTEM_HINT,
VISION_TOOL_NAME,
analyze_image_with_vision_model,
config_with_vision_resource,
vision_system_hint,
vision_tool_description,
)
from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
@@ -407,12 +408,13 @@ class WorkflowRealtimeController:
return RealtimeActivation(continue_response=generate)
def _agent_prompt(self, node_id: str) -> str:
stage = self._engine.agent_stage_config(node_id)
prompt = (
f"{self._engine.prompt_for(node_id, self._store)}\n\n"
f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
)
if self._engine.agent_stage_config(node_id).vision_enabled:
prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
if stage.vision_enabled:
prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
return prompt
def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
@@ -603,6 +605,7 @@ class WorkflowRealtimeController:
try:
if vision_cfg is not None:
vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
observation = await analyze_image_with_vision_model(
vision_cfg,
frame,
@@ -634,7 +637,7 @@ class WorkflowRealtimeController:
return (
RealtimeTool(
name=VISION_TOOL_NAME,
description="获取用户摄像头的当前画面,并回答一个视觉问题。",
description=vision_tool_description(stage.vision_focus_prompt),
properties={
"question": {
"type": "string",