238 lines
8.8 KiB
Python
238 lines
8.8 KiB
Python
"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
from io import BytesIO
|
|
|
|
from models import AssistantConfig, RuntimeModelResource
|
|
from openai import AsyncOpenAI
|
|
from PIL import Image
|
|
from pipecat.frames.frames import UserImageRawFrame
|
|
|
|
|
|
VISION_TOOL_NAME = "fetch_user_image"
|
|
VISION_SYSTEM_HINT = (
|
|
"当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
|
|
"环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
|
|
"再基于画面回答。"
|
|
)
|
|
VISION_ANALYSIS_SYSTEM_PROMPT = (
|
|
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
|
"如果画面不足以判断，请明确说明不确定。"
|
|
)
|
|
VISION_FOCUS_GENERATION_SYSTEM_PROMPT = (
|
|
"你负责把对话助手提示词整理成一段可编辑的视觉关注目标。"
|
|
"只提取图片中能够直接观察的对象、属性、状态、输出要求和不确定性边界；"
|
|
"不要复制对话流程、工具调用规则或结束条件，不要要求根据画面推断身份、责任、"
|
|
"意图等不可直接观察的信息。使用简洁中文，控制在 100 至 500 字，只输出关注目标正文。"
|
|
)
|
|
|
|
|
|
def _require(value: str, label: str) -> str:
|
|
if value:
|
|
return value
|
|
raise ValueError(f"缺少模型资源配置: {label}")
|
|
|
|
|
|
def vision_system_hint(focus_prompt: str = "") -> str:
|
|
"""Describe when to use vision and keep a configured business focus visible."""
|
|
|
|
focus = focus_prompt.strip()
|
|
if not focus:
|
|
return VISION_SYSTEM_HINT
|
|
return (
|
|
f"{VISION_SYSTEM_HINT}\n"
|
|
f"视觉关注目标（用于补充本次用户问题；冲突时以本次问题为准）：{focus}"
|
|
)
|
|
|
|
|
|
def vision_analysis_system_prompt(focus_prompt: str = "") -> str:
|
|
"""Keep the safety baseline fixed while appending an editable business focus."""
|
|
|
|
focus = focus_prompt.strip()
|
|
if not focus:
|
|
return VISION_ANALYSIS_SYSTEM_PROMPT
|
|
return (
|
|
f"{VISION_ANALYSIS_SYSTEM_PROMPT}\n\n"
|
|
f"[业务视觉关注目标]\n{focus}\n"
|
|
"该目标用于补充当前用户问题；如果二者范围不同，优先回答当前用户问题。"
|
|
)
|
|
|
|
|
|
def vision_tool_description(focus_prompt: str = "") -> str:
|
|
base = (
|
|
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
|
|
"人/物品/环境状态或需要视觉判断时调用。"
|
|
)
|
|
focus = " ".join(focus_prompt.split())
|
|
return f"{base} 当前业务重点：{focus[:600]}" if focus else base
|
|
|
|
|
|
async def generate_vision_focus_prompt(
|
|
resource: RuntimeModelResource,
|
|
source_prompt: str,
|
|
) -> str:
|
|
"""Generate one editable focus draft with an existing LLM resource."""
|
|
|
|
if resource.interface_type not in {"openai-llm", "dashscope-llm"}:
|
|
raise ValueError(f"当前模型接口不支持提示词生成: {resource.interface_type}")
|
|
source = source_prompt.strip()
|
|
if not source:
|
|
raise ValueError("助手提示词不能为空")
|
|
values = resource.values or {}
|
|
secrets = resource.secrets or {}
|
|
extra_body = values.get("extraBody")
|
|
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
|
client = AsyncOpenAI(
|
|
api_key=_require(str(secrets.get("apiKey") or ""), "LLM apiKey"),
|
|
base_url=_require(str(values.get("apiUrl") or ""), "LLM apiUrl"),
|
|
)
|
|
try:
|
|
response = await client.chat.completions.create(
|
|
model=_require(str(values.get("modelId") or ""), "LLM modelId"),
|
|
messages=[
|
|
{"role": "system", "content": VISION_FOCUS_GENERATION_SYSTEM_PROMPT},
|
|
{"role": "user", "content": source},
|
|
],
|
|
**extra,
|
|
)
|
|
finally:
|
|
await client.close()
|
|
content = response.choices[0].message.content if response.choices else ""
|
|
result = (
|
|
content.strip()
|
|
if isinstance(content, str)
|
|
else str(content or "").strip()
|
|
)
|
|
if not result:
|
|
raise ValueError("模型没有生成有效的视觉关注目标")
|
|
if result.startswith("```") and result.endswith("```"):
|
|
result = result[3:-3].strip()
|
|
if result.startswith("text"):
|
|
result = result[4:].strip()
|
|
return result[:4000]
|
|
|
|
|
|
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
|
|
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
|
|
if not frame.format:
|
|
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
|
|
buffer = BytesIO()
|
|
Image.frombytes(frame.format, frame.size, frame.image).save(
|
|
buffer,
|
|
format="JPEG",
|
|
quality=85,
|
|
)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def image_data_uri(frame: UserImageRawFrame) -> str:
|
|
"""Encode one Pipecat camera frame for a vision chat-completion request."""
|
|
encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
|
|
return f"data:image/jpeg;base64,{encoded}"
|
|
|
|
|
|
def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
|
|
"""Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
|
|
|
|
try:
|
|
image = Image.open(BytesIO(data)).convert("RGB")
|
|
image.load()
|
|
except (OSError, ValueError) as exc:
|
|
raise ValueError("上传图片无法解码") from exc
|
|
return UserImageRawFrame(
|
|
image=image.tobytes(),
|
|
size=image.size,
|
|
format="RGB",
|
|
)
|
|
|
|
|
|
async def analyze_image_with_vision_model(
|
|
cfg: AssistantConfig,
|
|
frame: UserImageRawFrame,
|
|
question: str,
|
|
) -> str:
|
|
"""Analyze one frame with the configured independent vision LLM."""
|
|
if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
|
|
raise ValueError(
|
|
f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
|
|
)
|
|
|
|
data_uri = await asyncio.to_thread(image_data_uri, frame)
|
|
extra_body = cfg.vision_llm_values.get("extraBody")
|
|
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
|
client = AsyncOpenAI(
|
|
api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
|
|
base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
|
|
)
|
|
try:
|
|
response = await client.chat.completions.create(
|
|
model=_require(cfg.vision_model, "Vision LLM modelId"),
|
|
messages=[
|
|
{
|
|
"role": "system",
|
|
"content": vision_analysis_system_prompt(
|
|
cfg.vision_focus_prompt
|
|
),
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": question},
|
|
{"type": "image_url", "image_url": {"url": data_uri}},
|
|
],
|
|
},
|
|
],
|
|
**extra,
|
|
)
|
|
finally:
|
|
await client.close()
|
|
|
|
content = response.choices[0].message.content if response.choices else ""
|
|
if isinstance(content, str):
|
|
return content.strip()
|
|
return str(content or "").strip()
|
|
|
|
|
|
def config_with_vision_resource(
|
|
cfg: AssistantConfig,
|
|
resource: RuntimeModelResource,
|
|
) -> AssistantConfig:
|
|
"""Create the config view used to analyze one Workflow camera frame."""
|
|
if resource.capability != "LLM":
|
|
raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
|
|
if not resource.support_image_input:
|
|
raise ValueError(f"视觉模型不支持图片输入:{resource.id}")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
values = resource.values or {}
|
|
secrets = resource.secrets or {}
|
|
result.vision_model_resource_id = resource.id
|
|
result.vision_model = str(values.get("modelId") or "")
|
|
result.vision_llm_interface_type = resource.interface_type
|
|
result.vision_llm_values = values
|
|
result.vision_llm_secrets = secrets
|
|
result.vision_llm_support_image_input = resource.support_image_input
|
|
result.vision_llm_api_key = str(secrets.get("apiKey") or "")
|
|
result.vision_llm_base_url = str(values.get("apiUrl") or "")
|
|
return result
|
|
|
|
|
|
def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
|
|
"""Legacy fallback when a Workflow LLM is not present in the resource map."""
|
|
if not cfg.llm_support_image_input:
|
|
raise ValueError("当前大语言模型不支持图片输入")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
result.vision_model_resource_id = None
|
|
result.vision_model = cfg.model
|
|
result.vision_llm_interface_type = cfg.llm_interface_type
|
|
result.vision_llm_values = dict(cfg.llm_values)
|
|
result.vision_llm_secrets = dict(cfg.llm_secrets)
|
|
result.vision_llm_support_image_input = cfg.llm_support_image_input
|
|
result.vision_llm_api_key = cfg.llm_api_key
|
|
result.vision_llm_base_url = cfg.llm_base_url
|
|
return result
|