Add support for image input in model resources and enhance related configurations
- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
This commit is contained in:
@@ -3,9 +3,8 @@
|
||||
/**
|
||||
* 摄像头预览:纯前端 getUserMedia 视频流,供调试抽屉的「视频流」视图使用。
|
||||
*
|
||||
* 与 use-voice-preview 分离:麦克风/语音会话走 WebRTC 到后端,摄像头目前只在
|
||||
* 本地预览。后续接入视觉理解管线时,把这条 video track 加进现有 PeerConnection
|
||||
* (pc.addTrack)并在后端 transport 打开 video_in_enabled 即可。
|
||||
* 与 use-voice-preview 分离:这里负责摄像头设备枚举与本地预览;开始视觉会话时,
|
||||
* 调用方会把这条 video track 加进语音 WebRTC PeerConnection。
|
||||
*/
|
||||
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
@@ -50,12 +49,12 @@ export function useCameraPreview() {
|
||||
setStream(null);
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async () => {
|
||||
if (streamRef.current) return;
|
||||
const start = useCallback(async (): Promise<MediaStream | null> => {
|
||||
if (streamRef.current) return streamRef.current;
|
||||
setError(null);
|
||||
if (!window.isSecureContext || !navigator.mediaDevices?.getUserMedia) {
|
||||
setError("当前页面无法访问摄像头,请通过 https 或 localhost 访问。");
|
||||
return;
|
||||
return null;
|
||||
}
|
||||
setStarting(true);
|
||||
try {
|
||||
@@ -66,21 +65,24 @@ export function useCameraPreview() {
|
||||
streamRef.current = media;
|
||||
setStream(media);
|
||||
void refreshCameras();
|
||||
return media;
|
||||
} catch (mediaError) {
|
||||
setError(cameraErrorMessage(mediaError));
|
||||
return null;
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}, [refreshCameras]);
|
||||
|
||||
const selectCamera = useCallback(
|
||||
async (id: string) => {
|
||||
async (id: string): Promise<MediaStream | null> => {
|
||||
deviceIdRef.current = id;
|
||||
setDeviceId(id);
|
||||
if (streamRef.current) {
|
||||
stop();
|
||||
await start();
|
||||
return await start();
|
||||
}
|
||||
return null;
|
||||
},
|
||||
[start, stop],
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user