feat: add configurable vision focus prompts
This commit is contained in:
@@ -31,10 +31,11 @@ from services.system_tools import state_update_properties, system_tool_kind
|
||||
from services.tool_executor import ToolExecutionError, ToolExecutor
|
||||
from services.tool_policy import policy_for_tool
|
||||
from services.vision import (
|
||||
VISION_SYSTEM_HINT,
|
||||
VISION_TOOL_NAME,
|
||||
analyze_image_with_vision_model,
|
||||
config_with_vision_resource,
|
||||
vision_system_hint,
|
||||
vision_tool_description,
|
||||
)
|
||||
from services.workflow.agent import EDGE_TOOL_STAGE_INSTRUCTION
|
||||
from services.workflow.models import WorkflowRuntimeState, WorkflowStatus
|
||||
@@ -407,12 +408,13 @@ class WorkflowRealtimeController:
|
||||
return RealtimeActivation(continue_response=generate)
|
||||
|
||||
def _agent_prompt(self, node_id: str) -> str:
|
||||
stage = self._engine.agent_stage_config(node_id)
|
||||
prompt = (
|
||||
f"{self._engine.prompt_for(node_id, self._store)}\n\n"
|
||||
f"[工作流执行规则]\n{EDGE_TOOL_STAGE_INSTRUCTION}"
|
||||
)
|
||||
if self._engine.agent_stage_config(node_id).vision_enabled:
|
||||
prompt = f"{prompt}\n{VISION_SYSTEM_HINT}"
|
||||
if stage.vision_enabled:
|
||||
prompt = f"{prompt}\n{vision_system_hint(stage.vision_focus_prompt)}"
|
||||
return prompt
|
||||
|
||||
def _build_agent_tools(self, node_id: str) -> list[RealtimeTool]:
|
||||
@@ -603,6 +605,7 @@ class WorkflowRealtimeController:
|
||||
|
||||
try:
|
||||
if vision_cfg is not None:
|
||||
vision_cfg.vision_focus_prompt = stage.vision_focus_prompt
|
||||
observation = await analyze_image_with_vision_model(
|
||||
vision_cfg,
|
||||
frame,
|
||||
@@ -634,7 +637,7 @@ class WorkflowRealtimeController:
|
||||
return (
|
||||
RealtimeTool(
|
||||
name=VISION_TOOL_NAME,
|
||||
description="获取用户摄像头的当前画面,并回答一个视觉问题。",
|
||||
description=vision_tool_description(stage.vision_focus_prompt),
|
||||
properties={
|
||||
"question": {
|
||||
"type": "string",
|
||||
|
||||
Reference in New Issue
Block a user