Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input.
- Update the backend models, schemas, and database seed scripts to accommodate the new field.
- Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling.
- Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support.
- Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
This commit is contained in:
Xin Wang
2026-07-07 19:41:45 +08:00
parent 32a52d318a
commit c5db918830
18 changed files with 411 additions and 116 deletions
+10 -8
View File
@@ -3,9 +3,8 @@
/**
* 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
*
* 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
* 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
* (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
* 与 use-voice-preview 分离:这里负责摄像头设备枚举与本地预览;开始视觉会话时,
* 调用方会把这条 video track 加进语音 WebRTC PeerConnection。
*/
import { useCallback, useEffect, useRef, useState } from "react";
@@ -50,12 +49,12 @@ export function useCameraPreview() {
setStream(null);
}, []);
const start = useCallback(async () => {
if (streamRef.current) return;
const start = useCallback(async (): Promise<MediaStream | null> => {
if (streamRef.current) return streamRef.current;
setError(null);
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
setError("当前页面无法访问摄像头,请通过 https 或 localhost 访问。");
return;
return null;
}
setStarting(true);
try {
@@ -66,21 +65,24 @@ export function useCameraPreview() {
streamRef.current = media;
setStream(media);
void refreshCameras();
return media;
} catch (mediaError) {
setError(cameraErrorMessage(mediaError));
return null;
} finally {
setStarting(false);
}
}, [refreshCameras]);
const selectCamera = useCallback(
async (id: string) => {
async (id: string): Promise<MediaStream | null> => {
deviceIdRef.current = id;
setDeviceId(id);
if (streamRef.current) {
stop();
await start();
return await start();
}
return null;
},
[start, stop],
);