feat: persist photos and enrich conversation history

frontend/src/hooks/use-photo-capture-tool.ts

@@ -19,8 +19,64 @@ const PHOTO_BUTTON_DEFINITION = {
   ],
 } as const;
 
+const MAX_PREVIEW_EDGE = 1280;
+
+async function capturePreviewImage(stream: MediaStream | null): Promise<string> {
+  const track = stream?.getVideoTracks()[0];
+  if (!stream || !track || track.readyState !== "live") {
+    throw new Error("当前没有可用的摄像头画面");
+  }
+
+  const video = document.createElement("video");
+  video.muted = true;
+  video.playsInline = true;
+  video.srcObject = stream;
+  try {
+    if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
+      await new Promise<void>((resolve, reject) => {
+        const timeout = window.setTimeout(
+          () => reject(new Error("等待摄像头预览超时")),
+          2_000,
+        );
+        video.addEventListener(
+          "loadeddata",
+          () => {
+            window.clearTimeout(timeout);
+            resolve();
+          },
+          { once: true },
+        );
+      });
+    }
+    await video.play();
+
+    const sourceWidth = video.videoWidth || track.getSettings().width || 0;
+    const sourceHeight = video.videoHeight || track.getSettings().height || 0;
+    if (!sourceWidth || !sourceHeight) {
+      throw new Error("摄像头画面尺寸不可用");
+    }
+    const scale = Math.min(1, MAX_PREVIEW_EDGE / Math.max(sourceWidth, sourceHeight));
+    const canvas = document.createElement("canvas");
+    canvas.width = Math.max(1, Math.round(sourceWidth * scale));
+    canvas.height = Math.max(1, Math.round(sourceHeight * scale));
+    const context = canvas.getContext("2d");
+    if (!context) throw new Error("浏览器无法生成照片预览");
+    context.drawImage(video, 0, 0, canvas.width, canvas.height);
+    return canvas.toDataURL("image/jpeg", 0.85);
+  } finally {
+    video.pause();
+    video.srcObject = null;
+  }
+}
+
 export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
-  const { registerClientTool, sendUserInput, status } = preview;
+  const {
+    appendUserImage,
+    registerClientTool,
+    sendUserInput,
+    status,
+    videoStream,
+  } = preview;
   const [visible, setVisible] = useState(false);
   const [capturing, setCapturing] = useState(false);
   const [error, setError] = useState<string | null>(null);
@@ -52,7 +108,9 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
     setCapturing(true);
     setError(null);
     try {
-      await sendUserInput(
+      const timestamp = new Date().toISOString();
+      const imageUrl = await capturePreviewImage(videoStream);
+      const result = await sendUserInput(
         [
           {
             type: "input_image",
@@ -61,6 +119,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
         ],
         { runImmediately: true, interrupt: true },
       );
+      appendUserImage(result.inputId, imageUrl, timestamp);
     } catch (captureError) {
       setError(
         captureError instanceof Error ? captureError.message : "拍照提交失败",
@@ -68,7 +127,7 @@ export function usePhotoCaptureTool(preview: VoicePreview, enabled = true) {
     } finally {
       setCapturing(false);
     }
-  }, [capturing, sendUserInput, status]);
+  }, [appendUserImage, capturing, sendUserInput, status, videoStream]);
 
   return {
     visible,

frontend/src/hooks/use-voice-preview.ts

@@ -45,6 +45,14 @@ export type ChatMessage = {
   sequence: number;
   turnId?: string;
   streaming?: boolean;
+  attachments?: ChatAttachment[];
+};
+
+export type ChatAttachment = {
+  id: string;
+  type: "image";
+  url: string;
+  alt: string;
 };
 
 type AppMessage = Record<string, unknown> & { type?: string };
@@ -871,6 +879,34 @@ export function useVoicePreview(
     [],
   );
 
+  const appendUserImage = useCallback(
+    (inputId: string, imageUrl: string, timestamp: string) => {
+      messageSeqRef.current += 1;
+      const sequence = messageSeqRef.current;
+      setMessages((previous) =>
+        sortMessages([
+          ...previous,
+          {
+            id: `user-image-${inputId}`,
+            role: "user",
+            content: "",
+            timestamp,
+            sequence,
+            attachments: [
+              {
+                id: `image-${inputId}`,
+                type: "image",
+                url: imageUrl,
+                alt: "用户拍摄的照片",
+              },
+            ],
+          },
+        ]),
+      );
+    },
+    [],
+  );
+
   const updateSession = useCallback(
     ({
       dynamicVariables,
@@ -962,6 +998,7 @@ export function useVoicePreview(
     supportsOutputSelection,
     sendText,
     sendUserInput,
+    appendUserImage,
     updateSession,
     registerClientTool,
     connect,