"use client"; /** * 语音预览:把麦克风接到后端 /ws/voice(WebRTC 信令),听到助手实时回应。 * * 走原生 RTCPeerConnection + 一条 ws 信令通道,与后端 voice_webrtc.py 的约定对齐: * client → {type:"offer", payload:{pc_id, sdp, type, assistant_id}} * server → {type:"answer", payload:{pc_id, sdp, type}} * client → {type:"ice-candidate", payload:{pc_id, candidate:{...}}} * 音频本身走 WebRTC 媒体流(Opus),不经 ws;后端 TTS 帧从 ontrack 拿到直接播放。 * * 另开一条 data channel 与后端管线(pipeline.py)互通应用消息: * client → {type:"user-text", text} 文字输入(打断并触发新回复) * server → {type:"transcript", role, content, timestamp} 用户/助手最终转写(聊天记录) * * 纯本机(localhost)即可跑:localhost 是 secure context,麦克风可用,ws 用明文。 * 局域网/别的设备要 https+wss,见 deploy/README.md。 */ import { useCallback, useEffect, useRef, useState } from "react"; import { API_BASE, webrtcApi } from "@/lib/api"; export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed"; export type NetworkQuality = "unknown" | "good" | "fair" | "poor"; const NETWORK_SAMPLE_INTERVAL_MS = 2_000; const VIDEO_BITRATE_BY_QUALITY: Record< Exclude, number > = { good: 600_000, fair: 500_000, poor: 300_000, }; const INITIAL_VIDEO_BITRATE = VIDEO_BITRATE_BY_QUALITY.fair; type NetworkStatsSample = { packetsSent: number; packetsLost: number; }; type VideoNetworkMetrics = { packetsSent: number | null; packetsLost: number | null; roundTripTime: number | null; availableOutgoingBitrate: number | null; }; type ConnectOptions = { visionEnabled?: boolean; videoStream?: MediaStream | null; }; export type ChatMessage = { id: string; role: "user" | "assistant"; content: string; /** 后端给的 ISO 时间戳 */ timestamp: string; sequence: number; turnId?: string; streaming?: boolean; }; // http→ws、https→wss,自动跟随 API 基址(同源反代时也对) function wsBaseUrl(): string { const url = new URL(API_BASE, window.location.origin); url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; return url.toString().replace(/\/$/, ""); } function generatePcId(): string { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return ( "PC-" + Array.from(bytes) .map((b) => b.toString(16).padStart(2, "0")) .join("") ); } function errorMessage(error: unknown, fallback: string): string { if (error instanceof Error && error.message) return error.message; return fallback; } function messageOrder(message: ChatMessage): number { const timestamp = Date.parse(message.timestamp); return Number.isNaN(timestamp) ? Number.MAX_SAFE_INTEGER : timestamp; } function sortMessages(messages: ChatMessage[]): ChatMessage[] { return messages.sort( (a, b) => messageOrder(a) - messageOrder(b) || a.sequence - b.sequence, ); } function microphoneErrorMessage(error: unknown): string { if (error instanceof DOMException) { if (error.name === "NotAllowedError") { return "麦克风权限被拒绝,请在浏览器网站设置中允许 localhost 使用麦克风。"; } if (error.name === "NotFoundError") { return "未检测到可用麦克风,请连接或启用麦克风后重试。"; } if (error.name === "NotReadableError") { return "麦克风正被其他应用占用,或系统未允许浏览器访问麦克风。"; } } return errorMessage(error, "无法访问麦克风。"); } async function setVideoBitrate( sender: RTCRtpSender, bitrate: number, ): Promise { try { const parameters = sender.getParameters(); if (!parameters.encodings.length) parameters.encodings = [{}]; parameters.encodings[0].maxBitrate = bitrate; parameters.degradationPreference = "maintain-resolution"; await sender.setParameters(parameters); return true; } catch { // 不支持发送参数的浏览器继续使用 WebRTC 自带的拥塞控制。 return false; } } function readVideoNetworkMetrics(report: RTCStatsReport): VideoNetworkMetrics { const metrics: VideoNetworkMetrics = { packetsSent: null, packetsLost: null, roundTripTime: null, availableOutgoingBitrate: null, }; report.forEach((stat) => { const values = stat as RTCStats & Record; const kind = values.kind ?? values.mediaType; if (values.type === "outbound-rtp" && kind === "video") { if (typeof values.packetsSent === "number") { metrics.packetsSent = values.packetsSent; } } else if (values.type === "remote-inbound-rtp" && kind === "video") { if (typeof values.packetsLost === "number") { metrics.packetsLost = values.packetsLost; } if (typeof values.roundTripTime === "number") { metrics.roundTripTime = values.roundTripTime; } } else if ( values.type === "candidate-pair" && values.state === "succeeded" && (values.nominated === true || values.selected === true) ) { if (typeof values.currentRoundTripTime === "number") { metrics.roundTripTime = values.currentRoundTripTime; } if (typeof values.availableOutgoingBitrate === "number") { metrics.availableOutgoingBitrate = values.availableOutgoingBitrate; } } }); return metrics; } function classifyNetworkQuality( metrics: VideoNetworkMetrics, previous: NetworkStatsSample | null, ): NetworkQuality { const hasPacketSample = previous !== null && metrics.packetsSent !== null && metrics.packetsLost !== null; const sentDelta = hasPacketSample ? Math.max(0, metrics.packetsSent! - previous.packetsSent) : 0; const lostDelta = hasPacketSample ? Math.max(0, metrics.packetsLost! - previous.packetsLost) : 0; const lossRate = sentDelta > 0 ? lostDelta / sentDelta : null; const { roundTripTime: rtt, availableOutgoingBitrate: bandwidth } = metrics; if (lossRate === null && rtt === null && bandwidth === null) return "unknown"; if ( (lossRate !== null && lossRate >= 0.08) || (rtt !== null && rtt >= 0.45) || (bandwidth !== null && bandwidth < 350_000) ) { return "poor"; } if ( (lossRate !== null && lossRate >= 0.03) || (rtt !== null && rtt >= 0.25) || (bandwidth !== null && bandwidth < 550_000) ) { return "fair"; } return "good"; } export function useVoicePreview( assistantId: string | null, onNodeActive?: (nodeId: string | null) => void, ) { const [status, setStatus] = useState("idle"); const [error, setError] = useState(null); const [micWarning, setMicWarning] = useState(null); const [localStream, setLocalStream] = useState(null); // 远端(助手 TTS)媒体流:除挂到