feat: support uploaded images in debug voice preview

Allow paste/drag temporary image assets so vision turns work without a live camera frame.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/services/conversation_history.py

@@ -176,6 +176,8 @@ class ConversationRecorder:
         *,
         input_id: str,
         timestamp: object,
+        content: str = "",
+        source: str = "camera_capture",
         mime_type: str = "image/jpeg",
     ) -> None:
         """Persist a captured frame without delaying the realtime model turn."""
@@ -185,6 +187,8 @@ class ConversationRecorder:
                 data,
                 input_id=input_id,
                 timestamp=timestamp,
+                content=content,
+                source=source,
                 mime_type=mime_type,
             ),
             name=f"conversation-image:{self.session_id}:{input_id}",
@@ -198,6 +202,8 @@ class ConversationRecorder:
         *,
         input_id: str,
         timestamp: object,
+        content: str,
+        source: str,
         mime_type: str,
     ) -> None:
         message_id = f"msg_{uuid4().hex[:20]}"
@@ -221,11 +227,11 @@ class ConversationRecorder:
                             sequence=next_sequence,
                             role="user",
                             content_type="image",
-                            content="",
+                            content=content.strip(),
                             occurred_at=_parse_timestamp(timestamp),
                             extra={
                                 "input_id": input_id,
-                                "source": "camera_capture",
+                                "source": source,
                             },
                         )
                     )
@@ -238,7 +244,7 @@ class ConversationRecorder:
                             storage_uri=storage_uri(key),
                             mime_type=mime_type,
                             size_bytes=len(data),
-                            extra={"input_id": input_id},
+                            extra={"input_id": input_id, "source": source},
                         )
                     )
                     conversation = await db.get(ConversationSession, self.session_id)

backend/services/input_assets.py

@@ -0,0 +1,132 @@
+"""Short-lived uploaded images consumed by the realtime user-input protocol."""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import json
+import time
+from dataclasses import dataclass
+from io import BytesIO
+from uuid import uuid4
+
+from PIL import Image, ImageOps, UnidentifiedImageError
+
+import settings
+from services.object_storage import delete_object, get_object, put_object
+
+
+INPUT_ASSET_PREFIX = "conversation-inputs/"
+
+
+@dataclass(frozen=True)
+class StoredInputImage:
+    token: str
+    width: int
+    height: int
+    size_bytes: int
+
+
+def _b64encode(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
+
+
+def _b64decode(data: str) -> bytes:
+    padding = "=" * (-len(data) % 4)
+    return base64.urlsafe_b64decode(f"{data}{padding}".encode("ascii"))
+
+
+def _signature(payload: str) -> str:
+    digest = hmac.new(
+        settings.AUTH_SECRET_KEY.encode("utf-8"),
+        payload.encode("ascii"),
+        hashlib.sha256,
+    ).digest()
+    return _b64encode(digest)
+
+
+def _normalize_image(data: bytes) -> tuple[bytes, int, int]:
+    try:
+        source = Image.open(BytesIO(data))
+        width, height = source.size
+        if width <= 0 or height <= 0:
+            raise ValueError("图片尺寸无效")
+        if width * height > settings.INPUT_IMAGE_MAX_PIXELS:
+            raise ValueError("图片像素尺寸过大")
+        source.load()
+    except (UnidentifiedImageError, OSError) as exc:
+        raise ValueError("文件不是可识别的图片") from exc
+
+    image = ImageOps.exif_transpose(source)
+    image.thumbnail(
+        (settings.INPUT_IMAGE_MAX_EDGE, settings.INPUT_IMAGE_MAX_EDGE),
+        Image.Resampling.LANCZOS,
+    )
+    if image.mode in {"RGBA", "LA"} or "transparency" in image.info:
+        rgba = image.convert("RGBA")
+        background = Image.new("RGB", rgba.size, "white")
+        background.paste(rgba, mask=rgba.getchannel("A"))
+        image = background
+    else:
+        image = image.convert("RGB")
+
+    buffer = BytesIO()
+    image.save(buffer, format="JPEG", quality=85, optimize=True)
+    return buffer.getvalue(), image.width, image.height
+
+
+def store_input_image(data: bytes) -> StoredInputImage:
+    normalized, width, height = _normalize_image(data)
+    key = f"{INPUT_ASSET_PREFIX}{uuid4().hex}.jpg"
+    put_object(key, normalized, "image/jpeg")
+    expires_at = int(time.time()) + settings.INPUT_IMAGE_TOKEN_TTL_SECONDS
+    encoded_payload = _b64encode(
+        json.dumps(
+            {"key": key, "exp": expires_at},
+            separators=(",", ":"),
+        ).encode("utf-8")
+    )
+    token = f"{encoded_payload}.{_signature(encoded_payload)}"
+    return StoredInputImage(
+        token=token,
+        width=width,
+        height=height,
+        size_bytes=len(normalized),
+    )
+
+
+def _key_from_token(token: str) -> str:
+    if not token or "." not in token:
+        raise ValueError("图片附件 token 无效")
+    encoded_payload, signature = token.rsplit(".", 1)
+    if not hmac.compare_digest(_signature(encoded_payload), signature):
+        raise ValueError("图片附件 token 签名无效")
+    try:
+        payload = json.loads(_b64decode(encoded_payload))
+        key = str(payload.get("key") or "")
+        expires_at = int(payload.get("exp") or 0)
+    except (ValueError, TypeError, json.JSONDecodeError) as exc:
+        raise ValueError("图片附件 token 内容无效") from exc
+    if expires_at < int(time.time()):
+        raise ValueError("图片附件已过期，请重新添加")
+    if not key.startswith(INPUT_ASSET_PREFIX):
+        raise ValueError("图片附件存储位置无效")
+    return key
+
+
+def consume_input_image(token: str) -> bytes:
+    """Read a signed image once and remove its temporary object."""
+
+    key = _key_from_token(token)
+    data = get_object(key)
+    try:
+        delete_object(key)
+    except Exception:
+        # Consumption succeeded. A stale temporary object must not break the turn.
+        pass
+    return data
+
+
+def discard_input_image(token: str) -> None:
+    delete_object(_key_from_token(token))

backend/services/pipecat/pipeline.py

@@ -27,6 +27,7 @@ from services.pipecat.service_factory import (
 )
 from db.session import SessionLocal
 from services.knowledge import search as search_knowledge
+from services.input_assets import consume_input_image
 from services.client_tools import ClientToolBroker
 from services.tool_policy import policy_for_tool
 from services.vision import (
@@ -36,6 +37,7 @@ from services.vision import (
     config_with_main_llm_as_vision,
     config_with_vision_resource,
     image_data_uri,
+    image_frame_from_jpeg,
     image_jpeg_bytes,
 )
 from services.workflow_engine import WorkflowEngine
@@ -734,7 +736,7 @@ async def run_pipeline(
     greeting = await brain.greeting(cfg)
 
     async def submit_user_input(value: UserInput) -> None:
-        if not value.has_camera_frame:
+        if not value.has_image:
             if not value.run_immediately:
                 brain.record_user_message(value.text)
             await worker.queue_frame(
@@ -749,28 +751,49 @@ async def run_pipeline(
             raise ValueError("P0 图片输入必须立即触发回复")
         if not vision_enabled:
             raise ValueError("当前助手未启用视觉能力")
-        user_id = vision_state.get("client_id")
-        if not user_id:
-            raise ValueError("当前没有可用的摄像头视频流")
         native_vision = active_vision_uses_main_llm()
         analysis_cfg = None if native_vision else active_vision_config()
 
-        request = UserImageRequestFrame(
-            user_id=user_id,
-            text=value.prompt_text,
-            append_to_context=False,
-        )
-        try:
-            image_frame = await vision_capture.request_image(llm, request)
-        except asyncio.TimeoutError as exc:
-            raise ValueError("等待摄像头视频帧超时") from exc
+        if value.image_asset_token:
+            try:
+                image_bytes = await asyncio.to_thread(
+                    consume_input_image,
+                    value.image_asset_token,
+                )
+                image_frame = await asyncio.to_thread(
+                    image_frame_from_jpeg,
+                    image_bytes,
+                )
+            except ValueError:
+                raise
+            except Exception as exc:
+                raise ValueError("上传图片暂时无法读取") from exc
+        else:
+            user_id = vision_state.get("client_id")
+            if not user_id:
+                raise ValueError("当前没有可用的摄像头视频流")
+            request = UserImageRequestFrame(
+                user_id=user_id,
+                text=value.prompt_text,
+                append_to_context=False,
+            )
+            try:
+                image_frame = await vision_capture.request_image(llm, request)
+            except asyncio.TimeoutError as exc:
+                raise ValueError("等待摄像头视频帧超时") from exc
+            image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
 
         if recorder:
-            image_bytes = await asyncio.to_thread(image_jpeg_bytes, image_frame)
             recorder.record_image_later(
                 image_bytes,
                 input_id=value.input_id,
                 timestamp=time_now_iso8601(),
+                content=value.text,
+                source=(
+                    "uploaded_asset"
+                    if value.image_asset_token
+                    else "camera_capture"
+                ),
             )
 
         if native_vision:

backend/services/pipecat/pipeline_events.py

@@ -149,11 +149,12 @@ def bind_cascade_pipeline_events(
 
     @text_input.event_handler("on_user_input")
     async def on_user_input(_processor, user_input: UserInput):
-        await queue_transcript(
-            "user",
-            user_input.transcript_text,
-            time_now_iso8601(),
-        )
+        if not user_input.has_image:
+            await queue_transcript(
+                "user",
+                user_input.transcript_text,
+                time_now_iso8601(),
+            )
         if user_input.run_immediately and user_input.interrupt:
             pending_user_inputs.append(user_input)
             return

backend/services/pipecat/processors.py

@@ -50,9 +50,14 @@ class UserInput:
     input_id: str
     text: str
     has_camera_frame: bool
+    image_asset_token: str | None
     run_immediately: bool
     interrupt: bool
 
+    @property
+    def has_image(self) -> bool:
+        return self.has_camera_frame or bool(self.image_asset_token)
+
     @property
     def prompt_text(self) -> str:
         if self.text:
@@ -61,10 +66,9 @@ class UserInput:
 
     @property
     def transcript_text(self) -> str:
-        # Images are represented as structured media records. A synthetic
-        # sentence would be shown as chat text and persisted as if the user had
-        # said it, while adding no visual information for the model.
-        return self.text
+        # Image turns are persisted as one structured media message, including
+        # any optional text caption. Do not create a duplicate transcript row.
+        return "" if self.has_image else self.text
 
 
 class UserInputError(ValueError):
@@ -155,6 +159,7 @@ def parse_user_input(message) -> UserInput | None:
 
     text = ""
     has_camera_frame = False
+    image_asset_token: str | None = None
     for part in parts:
         if not isinstance(part, dict):
             raise UserInputError("user-input part 格式不正确", input_id=input_id)
@@ -166,23 +171,30 @@ def parse_user_input(message) -> UserInput | None:
             if not text:
                 raise UserInputError("input_text 不能为空", input_id=input_id)
         elif part_type == "input_image":
-            if has_camera_frame:
+            if has_camera_frame or image_asset_token:
                 raise UserInputError("P0 只支持一张图片", input_id=input_id)
             source = part.get("source")
-            if (
-                not isinstance(source, dict)
-                or source.get("type") != "camera_frame"
-                or source.get("frame") != "current"
-            ):
+            if not isinstance(source, dict):
+                raise UserInputError("input_image source 格式不正确", input_id=input_id)
+            if source.get("type") == "camera_frame" and source.get("frame") == "current":
+                has_camera_frame = True
+            elif source.get("type") == "uploaded_asset":
+                token = str(source.get("asset_token") or "").strip()
+                if not token or len(token) > 4096:
+                    raise UserInputError(
+                        "上传图片缺少有效的 asset_token",
+                        input_id=input_id,
+                    )
+                image_asset_token = token
+            else:
                 raise UserInputError(
-                    "P0 只支持当前摄像头画面",
+                    "仅支持当前摄像头画面或已上传图片",
                     input_id=input_id,
                 )
-            has_camera_frame = True
         else:
             raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
 
-    if not text and not has_camera_frame:
+    if not text and not (has_camera_frame or image_asset_token):
         raise UserInputError("user-input 没有有效内容", input_id=input_id)
 
     options = message.get("options")
@@ -193,6 +205,7 @@ def parse_user_input(message) -> UserInput | None:
         input_id=input_id,
         text=text,
         has_camera_frame=has_camera_frame,
+        image_asset_token=image_asset_token,
         run_immediately=run_immediately,
         interrupt=interrupt,
     )
@@ -565,7 +578,7 @@ class RealtimeUserInputProcessor(FrameProcessor):
                 "当前工作流节点暂不接收用户输入",
             )
             return
-        if user_input.has_camera_frame:
+        if user_input.has_image:
             await self._emit_error(
                 user_input.input_id,
                 "Realtime 模式暂不支持图片输入",

backend/services/vision.py

@@ -49,6 +49,21 @@ def image_data_uri(frame: UserImageRawFrame) -> str:
     return f"data:image/jpeg;base64,{encoded}"
 
 
+def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
+    """Decode one normalized uploaded JPEG into Pipecat's raw image frame."""
+
+    try:
+        image = Image.open(BytesIO(data)).convert("RGB")
+        image.load()
+    except (OSError, ValueError) as exc:
+        raise ValueError("上传图片无法解码") from exc
+    return UserImageRawFrame(
+        image=image.tobytes(),
+        size=image.size,
+        format="RGB",
+    )
+
+
 async def analyze_image_with_vision_model(
     cfg: AssistantConfig,
     frame: UserImageRawFrame,