ai-video-fullstack/backend/services/vision.py

"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""

from __future__ import annotations

import asyncio
import base64
from io import BytesIO
from typing import Any

from models import (
    VISION_PROMPT_MODES,
    AssistantConfig,
    RuntimeModelResource,
    VisionPromptMode,
)
from openai import AsyncOpenAI
from PIL import Image
from pipecat.frames.frames import UserImageRawFrame


VISION_TOOL_NAME = "fetch_user_image"
VISION_SYSTEM_HINT = (
    "当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
    "环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
    "再基于画面回答。"
)
VISION_ANALYSIS_SYSTEM_PROMPT = (
    "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
    "如果画面不足以判断，请明确说明不确定。"
)


def _require(value: str, label: str) -> str:
    if value:
        return value
    raise ValueError(f"缺少模型资源配置: {label}")


def normalize_vision_prompt_mode(value: object) -> VisionPromptMode:
    """Return a supported mode while keeping older saved configs compatible."""

    mode = str(value or "runtime")
    if mode == "fixed":
        return "fixed"
    if mode == "guided":
        return "guided"
    return "runtime"


def _compact_prompt(prompt: str, *, limit: int = 4000) -> str:
    return " ".join(prompt.split())[:limit]


def vision_system_hint(
    mode: VisionPromptMode = "runtime",
    configured_prompt: str = "",
) -> str:
    """Tell the conversation model how the camera tool forms its question."""

    prompt_mode = normalize_vision_prompt_mode(mode)
    prompt = _compact_prompt(configured_prompt)
    if prompt_mode == "fixed":
        return (
            f"{VISION_SYSTEM_HINT}\n"
            f"该工具始终使用固定视觉问题“{prompt}”，调用时不需要生成视觉问题。"
        )
    if prompt_mode == "guided":
        return (
            f"{VISION_SYSTEM_HINT}\n"
            "调用时请结合当前对话生成具体视觉问题，并遵守以下固定引导："
            f"{prompt}"
        )
    return (
        f"{VISION_SYSTEM_HINT}\n"
        "调用时请结合当前对话，生成一个具体、可由当前画面回答的视觉问题。"
    )


def vision_tool_schema_parts(
    mode: VisionPromptMode = "runtime",
    configured_prompt: str = "",
) -> tuple[str, dict[str, Any], list[str]]:
    """Build the tool contract for one configured visual-question mode."""

    base = (
        "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、"
        "人/物品/环境状态或需要视觉判断时调用。"
    )
    prompt_mode = normalize_vision_prompt_mode(mode)
    prompt = _compact_prompt(configured_prompt, limit=600)
    if prompt_mode == "fixed":
        return (
            f"{base} 每次都检查同一个固定问题：{prompt}",
            {},
            [],
        )

    question_description = "结合当前对话生成的、关于当前视频画面的具体问题。"
    description = base
    if prompt_mode == "guided":
        description = f"{base} 生成问题时必须遵守固定引导：{prompt}"
        question_description = f"结合当前对话生成具体视觉问题，并遵守固定引导：{prompt}"
    return (
        description,
        {"question": {"type": "string", "description": question_description}},
        ["question"],
    )


def resolve_vision_question(
    mode: VisionPromptMode,
    configured_prompt: str,
    arguments: dict[str, Any] | None,
) -> str:
    """Resolve the exact question sent with the captured image."""

    prompt_mode = normalize_vision_prompt_mode(mode)
    if prompt_mode == "fixed":
        question = configured_prompt.strip()
    else:
        question = str((arguments or {}).get("question") or "").strip()
    if not question:
        raise ValueError(
            "固定视觉提示词为空"
            if prompt_mode == "fixed"
            else "视觉问题为空"
        )
    return question


def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
    """Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
    if not frame.format:
        raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
    buffer = BytesIO()
    Image.frombytes(frame.format, frame.size, frame.image).save(
        buffer,
        format="JPEG",
        quality=85,
    )
    return buffer.getvalue()


def image_data_uri(frame: UserImageRawFrame) -> str:
    """Encode one Pipecat camera frame for a vision chat-completion request."""
    encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
    return f"data:image/jpeg;base64,{encoded}"


def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
    """Decode one normalized uploaded JPEG into Pipecat's raw image frame."""

    try:
        image = Image.open(BytesIO(data)).convert("RGB")
        image.load()
    except (OSError, ValueError) as exc:
        raise ValueError("上传图片无法解码") from exc
    return UserImageRawFrame(
        image=image.tobytes(),
        size=image.size,
        format="RGB",
    )


async def analyze_image_with_vision_model(
    cfg: AssistantConfig,
    frame: UserImageRawFrame,
    question: str,
) -> str:
    """Analyze one frame with the configured independent vision LLM."""
    if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
        raise ValueError(
            f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
        )

    data_uri = await asyncio.to_thread(image_data_uri, frame)
    extra_body = cfg.vision_llm_values.get("extraBody")
    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
    client = AsyncOpenAI(
        api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
        base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
    )
    try:
        response = await client.chat.completions.create(
            model=_require(cfg.vision_model, "Vision LLM modelId"),
            messages=[
                {
                    "role": "system",
                    "content": VISION_ANALYSIS_SYSTEM_PROMPT,
                },
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": question},
                        {"type": "image_url", "image_url": {"url": data_uri}},
                    ],
                },
            ],
            **extra,
        )
    finally:
        await client.close()

    content = response.choices[0].message.content if response.choices else ""
    if isinstance(content, str):
        return content.strip()
    return str(content or "").strip()


def config_with_vision_resource(
    cfg: AssistantConfig,
    resource: RuntimeModelResource,
) -> AssistantConfig:
    """Create the config view used to analyze one Workflow camera frame."""
    if resource.capability != "LLM":
        raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
    if not resource.support_image_input:
        raise ValueError(f"视觉模型不支持图片输入:{resource.id}")

    result = cfg.model_copy(deep=True)
    values = resource.values or {}
    secrets = resource.secrets or {}
    result.vision_model_resource_id = resource.id
    result.vision_model = str(values.get("modelId") or "")
    result.vision_llm_interface_type = resource.interface_type
    result.vision_llm_values = values
    result.vision_llm_secrets = secrets
    result.vision_llm_support_image_input = resource.support_image_input
    result.vision_llm_api_key = str(secrets.get("apiKey") or "")
    result.vision_llm_base_url = str(values.get("apiUrl") or "")
    return result


def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
    """Legacy fallback when a Workflow LLM is not present in the resource map."""
    if not cfg.llm_support_image_input:
        raise ValueError("当前大语言模型不支持图片输入")

    result = cfg.model_copy(deep=True)
    result.vision_model_resource_id = None
    result.vision_model = cfg.model
    result.vision_llm_interface_type = cfg.llm_interface_type
    result.vision_llm_values = dict(cfg.llm_values)
    result.vision_llm_secrets = dict(cfg.llm_secrets)
    result.vision_llm_support_image_input = cfg.llm_support_image_input
    result.vision_llm_api_key = cfg.llm_api_key
    result.vision_llm_base_url = cfg.llm_base_url
    return result