Implement camera preview functionality and enhance AssistantPage

- Add a new `useCameraPreview` hook for managing camera access and video stream handling. - Integrate camera preview capabilities into the AssistantPage, allowing users to toggle video stream visibility. - Introduce a new toggle for enabling visual understanding, which activates the video stream preview alongside voice functionalities. - Update the DebugDrawer component to support switching between chat and video views based on the visual understanding setting. - Refactor related components to accommodate the new camera features and improve user interaction during debugging.
2026-07-06 14:08:15 +08:00
parent 809b634420
commit d6b04d71a0
3 changed files with 531 additions and 181 deletions
@@ -0,0 +1,113 @@
+"use client";
+
+/**
+ * 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
+ *
+ * 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
+ * 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
+ * (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
+ */
+
+import { useCallback, useEffect, useRef, useState } from "react";
+
+function cameraErrorMessage(error: unknown): string {
+  if (error instanceof DOMException) {
+    if (error.name === "NotAllowedError")
+      return "摄像头权限被拒绝，请在浏览器网站设置中允许使用摄像头。";
+    if (error.name === "NotFoundError")
+      return "未检测到可用摄像头，请连接或启用摄像头后重试。";
+    if (error.name === "NotReadableError")
+      return "摄像头正被其他应用占用，或系统未允许浏览器访问摄像头。";
+  }
+  if (error instanceof Error && error.message) return error.message;
+  return "无法访问摄像头。";
+}
+
+export function useCameraPreview() {
+  const [stream, setStream] = useState<MediaStream | null>(null);
+  const [error, setError] = useState<string | null>(null);
+  const [starting, setStarting] = useState(false);
+  const [devices, setDevices] = useState<MediaDeviceInfo[]>([]);
+  const [deviceId, setDeviceId] = useState("");
+  const streamRef = useRef<MediaStream | null>(null);
+  const deviceIdRef = useRef("");
+
+  const active = Boolean(stream);
+
+  const refreshCameras = useCallback(async () => {
+    if (!navigator.mediaDevices?.enumerateDevices) return;
+    try {
+      const list = await navigator.mediaDevices.enumerateDevices();
+      setDevices(list.filter((d) => d.kind === "videoinput" && d.deviceId));
+    } catch {
+      /* 忽略枚举失败 */
+    }
+  }, []);
+
+  const stop = useCallback(() => {
+    streamRef.current?.getTracks().forEach((track) => track.stop());
+    streamRef.current = null;
+    setStream(null);
+  }, []);
+
+  const start = useCallback(async () => {
+    if (streamRef.current) return;
+    setError(null);
+    if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
+      setError("当前页面无法访问摄像头，请通过 https 或 localhost 访问。");
+      return;
+    }
+    setStarting(true);
+    try {
+      const id = deviceIdRef.current;
+      const media = await navigator.mediaDevices.getUserMedia({
+        video: id ? { deviceId: { exact: id } } : true,
+      });
+      streamRef.current = media;
+      setStream(media);
+      void refreshCameras();
+    } catch (mediaError) {
+      setError(cameraErrorMessage(mediaError));
+    } finally {
+      setStarting(false);
+    }
+  }, [refreshCameras]);
+
+  const selectCamera = useCallback(
+    async (id: string) => {
+      deviceIdRef.current = id;
+      setDeviceId(id);
+      if (streamRef.current) {
+        stop();
+        await start();
+      }
+    },
+    [start, stop],
+  );
+
+  useEffect(() => {
+    if (!navigator.mediaDevices?.enumerateDevices) return;
+    // eslint-disable-next-line react-hooks/set-state-in-effect
+    void refreshCameras();
+    navigator.mediaDevices.addEventListener("devicechange", refreshCameras);
+    return () => {
+      navigator.mediaDevices.removeEventListener("devicechange", refreshCameras);
+    };
+  }, [refreshCameras]);
+
+  useEffect(() => stop, [stop]);
+
+  return {
+    stream,
+    error,
+    starting,
+    devices,
+    deviceId,
+    active,
+    start,
+    stop,
+    selectCamera,
+  };
+}
+
+export type CameraPreview = ReturnType<typeof useCameraPreview>;
@@ -558,3 +558,5 @@ export function useVoicePreview(
    audioRef,
  };
 }
+
+export type VoicePreview = ReturnType<typeof useVoicePreview>;