fix: support vision prompt modes
This commit is contained in:
Xin Wang
@@ -38,8 +38,9 @@ from services.vision import (
|
||||
image_data_uri,
|
||||
image_frame_from_jpeg,
|
||||
image_jpeg_bytes,
|
||||
resolve_vision_question,
|
||||
vision_system_hint,
|
||||
vision_tool_description,
|
||||
vision_tool_schema_parts,
|
||||
)
|
||||
from services.workflow_engine import WorkflowEngine
|
||||
|
||||
@@ -349,7 +350,9 @@ async def run_pipeline(
|
||||
def with_vision_hint(text: str) -> str:
|
||||
hints = []
|
||||
if vision_enabled and cfg.type != "workflow":
|
||||
hints.append(vision_system_hint(cfg.vision_focus_prompt))
|
||||
hints.append(
|
||||
vision_system_hint(cfg.vision_prompt_mode, cfg.vision_prompt)
|
||||
)
|
||||
if cfg.knowledge_base_id:
|
||||
hints.append(
|
||||
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
|
||||
@@ -450,7 +453,8 @@ async def run_pipeline(
|
||||
workflow_vision_scope: dict[str, Any] = {
|
||||
"enabled": False,
|
||||
"vision_model_resource_id": None,
|
||||
"vision_focus_prompt": "",
|
||||
"vision_prompt_mode": "runtime",
|
||||
"vision_prompt": "",
|
||||
"llm_resource_id": None,
|
||||
}
|
||||
|
||||
@@ -458,16 +462,15 @@ async def run_pipeline(
|
||||
if cfg.type == "workflow":
|
||||
return _workflow_vision_uses_main_llm(cfg, workflow_vision_scope)
|
||||
return vision_native_mode
|
||||
vision_description, vision_properties, vision_required = vision_tool_schema_parts(
|
||||
cfg.vision_prompt_mode,
|
||||
cfg.vision_prompt,
|
||||
)
|
||||
vision_schema = FunctionSchema(
|
||||
name=VISION_TOOL_NAME,
|
||||
description=vision_tool_description(cfg.vision_focus_prompt),
|
||||
properties={
|
||||
"question": {
|
||||
"type": "string",
|
||||
"description": "用户关于当前视频画面的具体问题。",
|
||||
}
|
||||
},
|
||||
required=["question"],
|
||||
description=vision_description,
|
||||
properties=vision_properties,
|
||||
required=vision_required,
|
||||
)
|
||||
knowledge_schema = FunctionSchema(
|
||||
name=KNOWLEDGE_TOOL_NAME,
|
||||
@@ -498,7 +501,15 @@ async def run_pipeline(
|
||||
await params.result_callback({"status": "error", "message": "知识库检索暂时不可用"})
|
||||
|
||||
async def fetch_user_image(params: FunctionCallParams):
|
||||
question = str(params.arguments.get("question") or "请描述当前画面。")
|
||||
try:
|
||||
question = resolve_vision_question(
|
||||
cfg.vision_prompt_mode,
|
||||
cfg.vision_prompt,
|
||||
params.arguments,
|
||||
)
|
||||
except ValueError as exc:
|
||||
await params.result_callback({"status": "error", "message": str(exc)})
|
||||
return
|
||||
user_id = vision_state.get("client_id")
|
||||
if not user_id:
|
||||
await params.result_callback(
|
||||
@@ -576,14 +587,26 @@ async def run_pipeline(
|
||||
result = config_with_main_llm_as_vision(cfg)
|
||||
else:
|
||||
raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
|
||||
result.vision_focus_prompt = str(
|
||||
workflow_vision_scope.get("vision_focus_prompt") or ""
|
||||
result.vision_prompt_mode = str(
|
||||
workflow_vision_scope.get("vision_prompt_mode") or "runtime"
|
||||
)
|
||||
result.vision_prompt = str(
|
||||
workflow_vision_scope.get("vision_prompt") or ""
|
||||
).strip()
|
||||
return result
|
||||
|
||||
if cfg.type == "workflow" and vision_enabled:
|
||||
async def native_flow_fetch_user_image(params: FunctionCallParams) -> None:
|
||||
question = str(params.arguments.get("question") or "请描述当前画面。")
|
||||
try:
|
||||
vision_cfg = active_vision_config()
|
||||
question = resolve_vision_question(
|
||||
vision_cfg.vision_prompt_mode,
|
||||
vision_cfg.vision_prompt,
|
||||
params.arguments,
|
||||
)
|
||||
except ValueError as exc:
|
||||
await params.result_callback({"status": "error", "message": str(exc)})
|
||||
return
|
||||
user_id = vision_state.get("client_id")
|
||||
if not user_id:
|
||||
await params.result_callback(
|
||||
@@ -609,7 +632,15 @@ async def run_pipeline(
|
||||
"status": "disabled",
|
||||
"message": "当前 Agent 节点未开启视觉理解。",
|
||||
}
|
||||
question = str((args or {}).get("question") or "请描述当前画面。")
|
||||
vision_cfg = active_vision_config()
|
||||
try:
|
||||
question = resolve_vision_question(
|
||||
vision_cfg.vision_prompt_mode,
|
||||
vision_cfg.vision_prompt,
|
||||
args,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "message": str(exc)}
|
||||
user_id = vision_state.get("client_id")
|
||||
if not user_id:
|
||||
return {
|
||||
@@ -625,7 +656,7 @@ async def run_pipeline(
|
||||
try:
|
||||
frame = await vision_capture.request_image(llm, request)
|
||||
observation = await analyze_image_with_vision_model(
|
||||
active_vision_config(),
|
||||
vision_cfg,
|
||||
frame,
|
||||
question,
|
||||
)
|
||||
@@ -654,14 +685,19 @@ async def run_pipeline(
|
||||
active_vision_uses_main_llm,
|
||||
)
|
||||
|
||||
workflow_vision_function = FlowsFunctionSchema(
|
||||
name=VISION_TOOL_NAME,
|
||||
description=vision_schema.description,
|
||||
properties=vision_schema.properties,
|
||||
required=vision_schema.required,
|
||||
handler=flow_fetch_user_image,
|
||||
cancel_on_interruption=True,
|
||||
)
|
||||
def workflow_vision_function(stage) -> FlowsFunctionSchema:
|
||||
description, properties, required = vision_tool_schema_parts(
|
||||
stage.vision_prompt_mode,
|
||||
stage.vision_prompt,
|
||||
)
|
||||
return FlowsFunctionSchema(
|
||||
name=VISION_TOOL_NAME,
|
||||
description=description,
|
||||
properties=properties,
|
||||
required=required,
|
||||
handler=flow_fetch_user_image,
|
||||
cancel_on_interruption=True,
|
||||
)
|
||||
|
||||
if vision_enabled and cfg.type != "workflow":
|
||||
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
|
||||
|
||||
Reference in New Issue
Block a user