feat: add configurable vision focus prompts

This commit is contained in:
Xin Wang
2026-08-12 17:29:30 +08:00
parent 7560f9a525
commit 09b07751b6
26 changed files with 477 additions and 20 deletions
+91 -1
View File
@@ -22,6 +22,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
"如果画面不足以判断,请明确说明不确定。"
)
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界;"
"不要复制对话流程、工具调用规则或结束条件,不要要求根据画面推断身份、责任、"
"意图等不可直接观察的信息。使用简洁中文,控制在 100 至 500 字,只输出关注目标正文。"
)
def _require(value: str, label: str) -> str:
@@ -30,6 +36,85 @@ def _require(value: str, label: str) -> str:
raise ValueError(f"缺少模型资源配置: {label}")
def vision_system_hint(focus_prompt: str = "") -> str:
"""Describe when to use vision and keep a configured business focus visible."""
focus = focus_prompt.strip()
if not focus:
return VISION_SYSTEM_HINT
return (
f"{VISION_SYSTEM_HINT}\n"
f"视觉关注目标(用于补充本次用户问题;冲突时以本次问题为准):{focus}"
)
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
"""Keep the safety baseline fixed while appending an editable business focus."""
focus = focus_prompt.strip()
if not focus:
return VISION_ANALYSIS_SYSTEM_PROMPT
return (
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
f"[业务视觉关注目标]\n{focus}\n"
"该目标用于补充当前用户问题;如果二者范围不同,优先回答当前用户问题。"
)
def vision_tool_description(focus_prompt: str = "") -> str:
base = (
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
"人/物品/环境状态或需要视觉判断时调用。"
)
focus = " ".join(focus_prompt.split())
return f"{base} 当前业务重点:{focus[:600]}" if focus else base
async def generate_vision_focus_prompt(
resource: RuntimeModelResource,
source_prompt: str,
) -> str:
"""Generate one editable focus draft with an existing LLM resource."""
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
source = source_prompt.strip()
if not source:
raise ValueError("助手提示词不能为空")
values = resource.values or {}
secrets = resource.secrets or {}
extra_body = values.get("extraBody")
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
client = AsyncOpenAI(
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
)
try:
response = await client.chat.completions.create(
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
messages=[
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
{"role": "user", "content": source},
],
**extra,
)
finally:
await client.close()
content = response.choices[0].message.content if response.choices else ""
result = (
content.strip()
if isinstance(content, str)
else str(content or "").strip()
)
if not result:
raise ValueError("模型没有生成有效的视觉关注目标")
if result.startswith("```") and result.endswith("```"):
result = result[3:-3].strip()
if result.startswith("text"):
result = result[4:].strip()
return result[:4000]
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
if not frame.format:
@@ -86,7 +171,12 @@ async def analyze_image_with_vision_model(
response = await client.chat.completions.create(
model=_require(cfg.vision_model, "Vision LLM modelId"),
messages=[
{"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
{
"role": "system",
"content": vision_analysis_system_prompt(
cfg.vision_focus_prompt
),
},
{
"role": "user",
"content": [