feat: add realtime workflow vision tools

backend/services/vision.py

@@ -1,8 +1,15 @@
-"""Shared vision capability metadata and Workflow model resolution."""
+"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""
 
 from __future__ import annotations
 
+import asyncio
+import base64
+from io import BytesIO
+
 from models import AssistantConfig, RuntimeModelResource
+from openai import AsyncOpenAI
+from PIL import Image
+from pipecat.frames.frames import UserImageRawFrame
 
 
 VISION_TOOL_NAME = "fetch_user_image"
@@ -17,6 +24,68 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
 )
 
 
+def _require(value: str, label: str) -> str:
+    if value:
+        return value
+    raise ValueError(f"缺少模型资源配置: {label}")
+
+
+def image_data_uri(frame: UserImageRawFrame) -> str:
+    """Encode one Pipecat camera frame for a vision chat-completion request."""
+    if not frame.format:
+        raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
+    buffer = BytesIO()
+    Image.frombytes(frame.format, frame.size, frame.image).save(
+        buffer,
+        format="JPEG",
+        quality=85,
+    )
+    encoded = base64.b64encode(buffer.getvalue()).decode("utf-8")
+    return f"data:image/jpeg;base64,{encoded}"
+
+
+async def analyze_image_with_vision_model(
+    cfg: AssistantConfig,
+    frame: UserImageRawFrame,
+    question: str,
+) -> str:
+    """Analyze one frame with the configured independent vision LLM."""
+    if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
+        raise ValueError(
+            f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
+        )
+
+    data_uri = await asyncio.to_thread(image_data_uri, frame)
+    extra_body = cfg.vision_llm_values.get("extraBody")
+    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
+    client = AsyncOpenAI(
+        api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
+        base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
+    )
+    try:
+        response = await client.chat.completions.create(
+            model=_require(cfg.vision_model, "Vision LLM modelId"),
+            messages=[
+                {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": question},
+                        {"type": "image_url", "image_url": {"url": data_uri}},
+                    ],
+                },
+            ],
+            **extra,
+        )
+    finally:
+        await client.close()
+
+    content = response.choices[0].message.content if response.choices else ""
+    if isinstance(content, str):
+        return content.strip()
+    return str(content or "").strip()
+
+
 def config_with_vision_resource(
     cfg: AssistantConfig,
     resource: RuntimeModelResource,