fix: support vision prompt modes
This commit is contained in:
Xin Wang
+85
-74
@@ -5,8 +5,14 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from typing import Any
|
||||
|
||||
from models import AssistantConfig, RuntimeModelResource
|
||||
from models import (
|
||||
VISION_PROMPT_MODES,
|
||||
AssistantConfig,
|
||||
RuntimeModelResource,
|
||||
VisionPromptMode,
|
||||
)
|
||||
from openai import AsyncOpenAI
|
||||
from PIL import Image
|
||||
from pipecat.frames.frames import UserImageRawFrame
|
||||
@@ -22,12 +28,6 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
|
||||
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
||||
"如果画面不足以判断，请明确说明不确定。"
|
||||
)
|
||||
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
|
||||
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
|
||||
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
|
||||
"不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
|
||||
"意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
|
||||
)
|
||||
|
||||
|
||||
def _require(value: str, label: str) -> str:
|
||||
@@ -36,83 +36,96 @@ def _require(value: str, label: str) -> str:
|
||||
raise ValueError(f"缺少模型资源配置: {label}")
|
||||
|
||||
|
||||
def vision_system_hint(focus_prompt: str = "") -> str:
|
||||
"""Describe when to use vision and keep a configured business focus visible."""
|
||||
def normalize_vision_prompt_mode(value: object) -> VisionPromptMode:
|
||||
"""Return a supported mode while keeping older saved configs compatible."""
|
||||
|
||||
focus = focus_prompt.strip()
|
||||
if not focus:
|
||||
return VISION_SYSTEM_HINT
|
||||
mode = str(value or "runtime")
|
||||
if mode == "fixed":
|
||||
return "fixed"
|
||||
if mode == "guided":
|
||||
return "guided"
|
||||
return "runtime"
|
||||
|
||||
|
||||
def _compact_prompt(prompt: str, *, limit: int = 4000) -> str:
|
||||
return " ".join(prompt.split())[:limit]
|
||||
|
||||
|
||||
def vision_system_hint(
|
||||
mode: VisionPromptMode = "runtime",
|
||||
configured_prompt: str = "",
|
||||
) -> str:
|
||||
"""Tell the conversation model how the camera tool forms its question."""
|
||||
|
||||
prompt_mode = normalize_vision_prompt_mode(mode)
|
||||
prompt = _compact_prompt(configured_prompt)
|
||||
if prompt_mode == "fixed":
|
||||
return (
|
||||
f"{VISION_SYSTEM_HINT}\n"
|
||||
f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。"
|
||||
)
|
||||
if prompt_mode == "guided":
|
||||
return (
|
||||
f"{VISION_SYSTEM_HINT}\n"
|
||||
"调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导："
|
||||
f"{prompt}"
|
||||
)
|
||||
return (
|
||||
f"{VISION_SYSTEM_HINT}\n"
|
||||
f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
|
||||
"调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。"
|
||||
)
|
||||
|
||||
|
||||
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
|
||||
"""Keep the safety baseline fixed while appending an editable business focus."""
|
||||
def vision_tool_schema_parts(
|
||||
mode: VisionPromptMode = "runtime",
|
||||
configured_prompt: str = "",
|
||||
) -> tuple[str, dict[str, Any], list[str]]:
|
||||
"""Build the tool contract for one configured visual-question mode."""
|
||||
|
||||
focus = focus_prompt.strip()
|
||||
if not focus:
|
||||
return VISION_ANALYSIS_SYSTEM_PROMPT
|
||||
return (
|
||||
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
|
||||
f"[业务视觉关注目标]\n{focus}\n"
|
||||
"该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
|
||||
)
|
||||
|
||||
|
||||
def vision_tool_description(focus_prompt: str = "") -> str:
|
||||
base = (
|
||||
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
|
||||
"人/物品/环境状态或需要视觉判断时调用。"
|
||||
)
|
||||
focus = " ".join(focus_prompt.split())
|
||||
return f"{base} 当前业务重点：{focus[:600]}" if focus else base
|
||||
|
||||
|
||||
async def generate_vision_focus_prompt(
|
||||
resource: RuntimeModelResource,
|
||||
source_prompt: str,
|
||||
) -> str:
|
||||
"""Generate one editable focus draft with an existing LLM resource."""
|
||||
|
||||
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
|
||||
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
|
||||
source = source_prompt.strip()
|
||||
if not source:
|
||||
raise ValueError("助手提示词不能为空")
|
||||
values = resource.values or {}
|
||||
secrets = resource.secrets or {}
|
||||
extra_body = values.get("extraBody")
|
||||
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
||||
client = AsyncOpenAI(
|
||||
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
|
||||
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
|
||||
)
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
|
||||
messages=[
|
||||
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": source},
|
||||
],
|
||||
**extra,
|
||||
prompt_mode = normalize_vision_prompt_mode(mode)
|
||||
prompt = _compact_prompt(configured_prompt, limit=600)
|
||||
if prompt_mode == "fixed":
|
||||
return (
|
||||
f"{base} 每次都检查同一个固定问题：{prompt}",
|
||||
{},
|
||||
[],
|
||||
)
|
||||
finally:
|
||||
await client.close()
|
||||
content = response.choices[0].message.content if response.choices else ""
|
||||
result = (
|
||||
content.strip()
|
||||
if isinstance(content, str)
|
||||
else str(content or "").strip()
|
||||
|
||||
question_description = "结合当前对话生成的、关于当前视频画面的具体问题。"
|
||||
description = base
|
||||
if prompt_mode == "guided":
|
||||
description = f"{base} 生成问题时必须遵守固定引导：{prompt}"
|
||||
question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}"
|
||||
return (
|
||||
description,
|
||||
{"question": {"type": "string", "description": question_description}},
|
||||
["question"],
|
||||
)
|
||||
if not result:
|
||||
raise ValueError("模型没有生成有效的视觉关注目标")
|
||||
if result.startswith("```") and result.endswith("```"):
|
||||
result = result[3:-3].strip()
|
||||
if result.startswith("text"):
|
||||
result = result[4:].strip()
|
||||
return result[:4000]
|
||||
|
||||
|
||||
def resolve_vision_question(
|
||||
mode: VisionPromptMode,
|
||||
configured_prompt: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
) -> str:
|
||||
"""Resolve the exact question sent with the captured image."""
|
||||
|
||||
prompt_mode = normalize_vision_prompt_mode(mode)
|
||||
if prompt_mode == "fixed":
|
||||
question = configured_prompt.strip()
|
||||
else:
|
||||
question = str((arguments or {}).get("question") or "").strip()
|
||||
if not question:
|
||||
raise ValueError(
|
||||
"固定视觉提示词为空"
|
||||
if prompt_mode == "fixed"
|
||||
else "视觉问题为空"
|
||||
)
|
||||
return question
|
||||
|
||||
|
||||
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
|
||||
@@ -173,9 +186,7 @@ async def analyze_image_with_vision_model(
|
||||
messages=[
|
||||
{
|
||||
"role": "system",
|
||||
"content": vision_analysis_system_prompt(
|
||||
cfg.vision_focus_prompt
|
||||
),
|
||||
"content": VISION_ANALYSIS_SYSTEM_PROMPT,
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
Reference in New Issue
Block a user