feat: add configurable vision focus prompts
This commit is contained in:
@@ -22,6 +22,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
|
||||
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
||||
"如果画面不足以判断,请明确说明不确定。"
|
||||
)
|
||||
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
|
||||
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
|
||||
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界;"
|
||||
"不要复制对话流程、工具调用规则或结束条件,不要要求根据画面推断身份、责任、"
|
||||
"意图等不可直接观察的信息。使用简洁中文,控制在 100 至 500 字,只输出关注目标正文。"
|
||||
)
|
||||
|
||||
|
||||
def _require(value: str, label: str) -> str:
|
||||
@@ -30,6 +36,85 @@ def _require(value: str, label: str) -> str:
|
||||
raise ValueError(f"缺少模型资源配置: {label}")
|
||||
|
||||
|
||||
def vision_system_hint(focus_prompt: str = "") -> str:
|
||||
"""Describe when to use vision and keep a configured business focus visible."""
|
||||
|
||||
focus = focus_prompt.strip()
|
||||
if not focus:
|
||||
return VISION_SYSTEM_HINT
|
||||
return (
|
||||
f"{VISION_SYSTEM_HINT}\n"
|
||||
f"视觉关注目标(用于补充本次用户问题;冲突时以本次问题为准):{focus}"
|
||||
)
|
||||
|
||||
|
||||
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
|
||||
"""Keep the safety baseline fixed while appending an editable business focus."""
|
||||
|
||||
focus = focus_prompt.strip()
|
||||
if not focus:
|
||||
return VISION_ANALYSIS_SYSTEM_PROMPT
|
||||
return (
|
||||
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
|
||||
f"[业务视觉关注目标]\n{focus}\n"
|
||||
"该目标用于补充当前用户问题;如果二者范围不同,优先回答当前用户问题。"
|
||||
)
|
||||
|
||||
|
||||
def vision_tool_description(focus_prompt: str = "") -> str:
|
||||
base = (
|
||||
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
|
||||
"人/物品/环境状态或需要视觉判断时调用。"
|
||||
)
|
||||
focus = " ".join(focus_prompt.split())
|
||||
return f"{base} 当前业务重点:{focus[:600]}" if focus else base
|
||||
|
||||
|
||||
async def generate_vision_focus_prompt(
|
||||
resource: RuntimeModelResource,
|
||||
source_prompt: str,
|
||||
) -> str:
|
||||
"""Generate one editable focus draft with an existing LLM resource."""
|
||||
|
||||
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
|
||||
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
|
||||
source = source_prompt.strip()
|
||||
if not source:
|
||||
raise ValueError("助手提示词不能为空")
|
||||
values = resource.values or {}
|
||||
secrets = resource.secrets or {}
|
||||
extra_body = values.get("extraBody")
|
||||
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
||||
client = AsyncOpenAI(
|
||||
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
|
||||
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
|
||||
)
|
||||
try:
|
||||
response = await client.chat.completions.create(
|
||||
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
|
||||
messages=[
|
||||
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": source},
|
||||
],
|
||||
**extra,
|
||||
)
|
||||
finally:
|
||||
await client.close()
|
||||
content = response.choices[0].message.content if response.choices else ""
|
||||
result = (
|
||||
content.strip()
|
||||
if isinstance(content, str)
|
||||
else str(content or "").strip()
|
||||
)
|
||||
if not result:
|
||||
raise ValueError("模型没有生成有效的视觉关注目标")
|
||||
if result.startswith("```") and result.endswith("```"):
|
||||
result = result[3:-3].strip()
|
||||
if result.startswith("text"):
|
||||
result = result[4:].strip()
|
||||
return result[:4000]
|
||||
|
||||
|
||||
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
|
||||
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
|
||||
if not frame.format:
|
||||
@@ -86,7 +171,12 @@ async def analyze_image_with_vision_model(
|
||||
response = await client.chat.completions.create(
|
||||
model=_require(cfg.vision_model, "Vision LLM modelId"),
|
||||
messages=[
|
||||
{"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
|
||||
{
|
||||
"role": "system",
|
||||
"content": vision_analysis_system_prompt(
|
||||
cfg.vision_focus_prompt
|
||||
),
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
|
||||
Reference in New Issue
Block a user