338 lines
12 KiB
Python
338 lines
12 KiB
Python
"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import base64
|
|
from dataclasses import dataclass
|
|
from io import BytesIO
|
|
from typing import Any
|
|
|
|
from models import (
|
|
VISION_PROMPT_MODES,
|
|
AssistantConfig,
|
|
RuntimeModelResource,
|
|
VisionPromptMode,
|
|
)
|
|
from openai import AsyncOpenAI
|
|
from PIL import Image
|
|
from pipecat.frames.frames import UserImageRawFrame
|
|
|
|
|
|
VISION_TOOL_NAME = "fetch_user_image"
|
|
VISION_SYSTEM_HINT = (
|
|
"当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
|
|
"环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
|
|
"再基于画面回答。"
|
|
)
|
|
VISION_ANALYSIS_SYSTEM_PROMPT = (
|
|
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
|
"如果画面不足以判断，请明确说明不确定。"
|
|
)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VisionInputResolution:
|
|
"""The active model path for one explicit user image turn."""
|
|
|
|
native: bool
|
|
config: AssistantConfig
|
|
|
|
|
|
def _require(value: str, label: str) -> str:
|
|
if value:
|
|
return value
|
|
raise ValueError(f"缺少模型资源配置: {label}")
|
|
|
|
|
|
def normalize_vision_prompt_mode(value: object) -> VisionPromptMode:
|
|
"""Return a supported vision prompt mode."""
|
|
|
|
mode = str(value or "runtime")
|
|
if mode == "fixed":
|
|
return "fixed"
|
|
if mode == "guided":
|
|
return "guided"
|
|
return "runtime"
|
|
|
|
|
|
def _compact_prompt(prompt: str, *, limit: int = 4000) -> str:
|
|
return " ".join(prompt.split())[:limit]
|
|
|
|
|
|
def vision_system_hint(
|
|
mode: VisionPromptMode = "runtime",
|
|
configured_prompt: str = "",
|
|
) -> str:
|
|
"""Tell the conversation model how the camera tool forms its question."""
|
|
|
|
prompt_mode = normalize_vision_prompt_mode(mode)
|
|
prompt = _compact_prompt(configured_prompt)
|
|
if prompt_mode == "fixed":
|
|
return (
|
|
f"{VISION_SYSTEM_HINT}\n"
|
|
f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。"
|
|
)
|
|
if prompt_mode == "guided":
|
|
return (
|
|
f"{VISION_SYSTEM_HINT}\n"
|
|
"调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导："
|
|
f"{prompt}"
|
|
)
|
|
return (
|
|
f"{VISION_SYSTEM_HINT}\n"
|
|
"调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。"
|
|
)
|
|
|
|
|
|
def vision_tool_schema_parts(
|
|
mode: VisionPromptMode = "runtime",
|
|
configured_prompt: str = "",
|
|
) -> tuple[str, dict[str, Any], list[str]]:
|
|
"""Build the tool contract for one configured visual-question mode."""
|
|
|
|
base = (
|
|
"获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
|
|
"人/物品/环境状态或需要视觉判断时调用。"
|
|
)
|
|
prompt_mode = normalize_vision_prompt_mode(mode)
|
|
prompt = _compact_prompt(configured_prompt, limit=600)
|
|
if prompt_mode == "fixed":
|
|
return (
|
|
f"{base} 每次都检查同一个固定问题：{prompt}",
|
|
{},
|
|
[],
|
|
)
|
|
|
|
question_description = "结合当前对话生成的、关于当前视频画面的具体问题。"
|
|
description = base
|
|
if prompt_mode == "guided":
|
|
description = f"{base} 生成问题时必须遵守固定引导：{prompt}"
|
|
question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}"
|
|
return (
|
|
description,
|
|
{"question": {"type": "string", "description": question_description}},
|
|
["question"],
|
|
)
|
|
|
|
|
|
def resolve_vision_question(
|
|
mode: VisionPromptMode,
|
|
configured_prompt: str,
|
|
arguments: dict[str, Any] | None,
|
|
) -> str:
|
|
"""Resolve the exact question sent with the captured image."""
|
|
|
|
prompt_mode = normalize_vision_prompt_mode(mode)
|
|
if prompt_mode == "fixed":
|
|
question = configured_prompt.strip()
|
|
else:
|
|
question = str((arguments or {}).get("question") or "").strip()
|
|
if not question:
|
|
raise ValueError(
|
|
"固定视觉提示词为空"
|
|
if prompt_mode == "fixed"
|
|
else "视觉问题为空"
|
|
)
|
|
return question
|
|
|
|
|
|
def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
|
|
"""Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
|
|
if not frame.format:
|
|
raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
|
|
buffer = BytesIO()
|
|
Image.frombytes(frame.format, frame.size, frame.image).save(
|
|
buffer,
|
|
format="JPEG",
|
|
quality=85,
|
|
)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def image_data_uri(frame: UserImageRawFrame) -> str:
|
|
"""Encode one Pipecat camera frame for a vision chat-completion request."""
|
|
encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
|
|
return f"data:image/jpeg;base64,{encoded}"
|
|
|
|
|
|
def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
|
|
"""Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
|
|
|
|
try:
|
|
image = Image.open(BytesIO(data)).convert("RGB")
|
|
image.load()
|
|
except (OSError, ValueError) as exc:
|
|
raise ValueError("上传图片无法解码") from exc
|
|
return UserImageRawFrame(
|
|
image=image.tobytes(),
|
|
size=image.size,
|
|
format="RGB",
|
|
)
|
|
|
|
|
|
async def analyze_image_with_vision_model(
|
|
cfg: AssistantConfig,
|
|
frame: UserImageRawFrame,
|
|
question: str,
|
|
) -> str:
|
|
"""Analyze one frame with the configured independent vision LLM."""
|
|
if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
|
|
raise ValueError(
|
|
f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
|
|
)
|
|
|
|
data_uri = await asyncio.to_thread(image_data_uri, frame)
|
|
extra_body = cfg.vision_llm_values.get("extraBody")
|
|
extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
|
|
client = AsyncOpenAI(
|
|
api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
|
|
base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
|
|
)
|
|
try:
|
|
response = await client.chat.completions.create(
|
|
model=_require(cfg.vision_model, "Vision LLM modelId"),
|
|
messages=[
|
|
{
|
|
"role": "system",
|
|
"content": VISION_ANALYSIS_SYSTEM_PROMPT,
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": question},
|
|
{"type": "image_url", "image_url": {"url": data_uri}},
|
|
],
|
|
},
|
|
],
|
|
**extra,
|
|
)
|
|
finally:
|
|
await client.close()
|
|
|
|
content = response.choices[0].message.content if response.choices else ""
|
|
if isinstance(content, str):
|
|
return content.strip()
|
|
return str(content or "").strip()
|
|
|
|
|
|
def config_with_vision_resource(
|
|
cfg: AssistantConfig,
|
|
resource: RuntimeModelResource,
|
|
) -> AssistantConfig:
|
|
"""Create the config view used to analyze one Workflow camera frame."""
|
|
if resource.capability != "LLM":
|
|
raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
|
|
if not resource.support_image_input:
|
|
raise ValueError(f"视觉模型不支持图片输入:{resource.id}")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
values = resource.values or {}
|
|
secrets = resource.secrets or {}
|
|
result.vision_model_resource_id = resource.id
|
|
result.vision_model = str(values.get("modelId") or "")
|
|
result.vision_llm_interface_type = resource.interface_type
|
|
result.vision_llm_values = values
|
|
result.vision_llm_secrets = secrets
|
|
result.vision_llm_support_image_input = resource.support_image_input
|
|
result.vision_llm_api_key = str(secrets.get("apiKey") or "")
|
|
result.vision_llm_base_url = str(values.get("apiUrl") or "")
|
|
return result
|
|
|
|
|
|
def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
|
|
"""Legacy fallback when a Workflow LLM is not present in the resource map."""
|
|
if not cfg.llm_support_image_input:
|
|
raise ValueError("当前大语言模型不支持图片输入")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
result.vision_model_resource_id = None
|
|
result.vision_model = cfg.model
|
|
result.vision_llm_interface_type = cfg.llm_interface_type
|
|
result.vision_llm_values = dict(cfg.llm_values)
|
|
result.vision_llm_secrets = dict(cfg.llm_secrets)
|
|
result.vision_llm_support_image_input = cfg.llm_support_image_input
|
|
result.vision_llm_api_key = cfg.llm_api_key
|
|
result.vision_llm_base_url = cfg.llm_base_url
|
|
return result
|
|
|
|
|
|
def resolve_vision_input(
|
|
cfg: AssistantConfig,
|
|
workflow_scope: dict[str, Any] | None = None,
|
|
) -> VisionInputResolution:
|
|
"""Resolve native versus auxiliary vision using the active runtime scope."""
|
|
|
|
if cfg.type != "workflow":
|
|
if not cfg.vision_enabled:
|
|
raise ValueError("当前助手未开启视觉能力")
|
|
if cfg.vision_model_resource_id:
|
|
if not cfg.vision_llm_support_image_input:
|
|
raise ValueError("当前视觉模型不支持图片输入")
|
|
return VisionInputResolution(native=False, config=cfg)
|
|
if not cfg.llm_support_image_input:
|
|
raise ValueError("当前大语言模型不支持图片输入")
|
|
return VisionInputResolution(native=True, config=cfg)
|
|
|
|
scope = workflow_scope or {}
|
|
if not scope.get("enabled"):
|
|
raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
|
|
vision_resource_id = str(scope.get("vision_model_resource_id") or "")
|
|
llm_resource_id = str(scope.get("llm_resource_id") or "")
|
|
resource_id = vision_resource_id or llm_resource_id
|
|
resource = cfg.workflow_model_resources.get(resource_id)
|
|
if resource is not None:
|
|
if not resource.support_image_input:
|
|
raise ValueError(f"当前视觉模型不支持图片输入: {resource.id}")
|
|
result = config_with_vision_resource(cfg, resource)
|
|
elif not vision_resource_id:
|
|
result = config_with_main_llm_as_vision(cfg)
|
|
else:
|
|
raise ValueError(f"视觉模型资源未加载: {vision_resource_id}")
|
|
|
|
result.vision_prompt_mode = normalize_vision_prompt_mode(
|
|
scope.get("vision_prompt_mode")
|
|
)
|
|
result.vision_prompt = str(scope.get("vision_prompt") or "").strip()
|
|
return VisionInputResolution(
|
|
native=not bool(vision_resource_id),
|
|
config=result,
|
|
)
|
|
|
|
|
|
async def build_image_user_message(
|
|
cfg: AssistantConfig,
|
|
image_bytes: bytes,
|
|
prompt_text: str,
|
|
*,
|
|
workflow_scope: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Build the same model-facing user message used by an explicit photo turn."""
|
|
|
|
question = prompt_text.strip() or "请根据用户刚提交的图片进行回复。"
|
|
frame = await asyncio.to_thread(image_frame_from_jpeg, image_bytes)
|
|
resolution = resolve_vision_input(cfg, workflow_scope)
|
|
if resolution.native:
|
|
data_uri = await asyncio.to_thread(image_data_uri, frame)
|
|
return {
|
|
"role": "user",
|
|
"content": [
|
|
{"type": "text", "text": question},
|
|
{"type": "image_url", "image_url": {"url": data_uri}},
|
|
],
|
|
}
|
|
|
|
observation = await analyze_image_with_vision_model(
|
|
resolution.config,
|
|
frame,
|
|
question,
|
|
)
|
|
return {
|
|
"role": "user",
|
|
"content": (
|
|
f"{question}\n\n"
|
|
"[视觉模型对用户刚提交图片的观察]\n"
|
|
f"{observation or '视觉模型没有返回有效观察结果。'}"
|
|
),
|
|
}
|