1543 lines
65 KiB
JavaScript
1543 lines
65 KiB
JavaScript
|
// ==UserScript==
|
|||
|
|
// @name 12345 AI Native Track Copilot
|
|||
|
|
// @namespace http://tampermonkey.net/
|
|||
|
|
// @version 1.0
|
|||
|
|
// @description Native-track only bridge: no WebAudio washing, no captureStream, no MediaStreamDestination.
|
|||
|
|
// @author You
|
|||
|
|
// @match *://yhy-yw.22sj.cn:*/*
|
|||
|
|
// @grant none
|
|||
|
|
// @run-at document-start
|
|||
|
|
// ==/UserScript==
|
|||
|
|
|
|||
|
|
(function () {
|
|||
|
|
'use strict';
|
|||
|
|
|
|||
|
|
const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
|
|||
|
|
const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio';
|
|||
|
|
const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video';
|
|||
|
|
|
|||
|
|
const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices);
|
|||
|
|
const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
|
|||
|
|
const originalAddTrack = RTCPeerConnection.prototype.addTrack;
|
|||
|
|
const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
|
|||
|
|
const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
|
|||
|
|
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
|
|||
|
|
|
|||
|
|
const remoteStreams = new Set();
|
|||
|
|
const ignoredTrackIds = new Set();
|
|||
|
|
const audioCandidates = new Map();
|
|||
|
|
const videoCandidates = new Map();
|
|||
|
|
const businessSenderCandidates = new Map();
|
|||
|
|
const localCaptureTrackIds = new Set();
|
|||
|
|
window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set();
|
|||
|
|
|
|||
|
|
let aiPc = null;
|
|||
|
|
let aiWs = null;
|
|||
|
|
let aiDc = null;
|
|||
|
|
let aiAudioSender = null;
|
|||
|
|
let aiVideoSender = null;
|
|||
|
|
|
|||
|
|
let businessAudioSender = null;
|
|||
|
|
let businessOriginalAudioTrack = null;
|
|||
|
|
|
|||
|
|
let remoteUserAudioTrack = null;
|
|||
|
|
let remoteUserVideoTrack = null;
|
|||
|
|
let aiResponseAudioTrack = null;
|
|||
|
|
let remoteUserAudioLocked = false;
|
|||
|
|
let remoteUserVideoLocked = false;
|
|||
|
|
|
|||
|
|
let isAiTakeover = false;
|
|||
|
|
let aiTrackAttachedToBusiness = false;
|
|||
|
|
let inputVuLevel = 0;
|
|||
|
|
let aiOutVuLevel = 0;
|
|||
|
|
let aiInputHealthTimer = null;
|
|||
|
|
let aiOutputHealthTimer = null;
|
|||
|
|
let assistantLine = null;
|
|||
|
|
let assistantBuffer = '';
|
|||
|
|
let inputRecorder = null;
|
|||
|
|
let inputRecordingChunks = [];
|
|||
|
|
let inputRecordingTrack = null;
|
|||
|
|
let inputRecordingStartedAt = 0;
|
|||
|
|
let videoRecorder = null;
|
|||
|
|
let videoRecordingChunks = [];
|
|||
|
|
let videoRecordingTrack = null;
|
|||
|
|
let videoRecordingStartedAt = 0;
|
|||
|
|
|
|||
|
|
const receiverStatsMemory = new Map();
|
|||
|
|
const senderStatsMemory = new Map();
|
|||
|
|
const inboundStatsMemory = new Map();
|
|||
|
|
const outboundStatsMemory = new Map();
|
|||
|
|
const businessOutboundStats = {
|
|||
|
|
bytesSent: 0,
|
|||
|
|
packetsSent: 0,
|
|||
|
|
bytesDelta: 0,
|
|||
|
|
packetsDelta: 0,
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
function log(message) {
|
|||
|
|
console.log(`[AI Native Copilot] ${message}`);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function readBoolSetting(key, fallback) {
|
|||
|
|
try {
|
|||
|
|
const value = localStorage.getItem(key);
|
|||
|
|
if (value === null) return fallback;
|
|||
|
|
return value === 'true';
|
|||
|
|
} catch {
|
|||
|
|
return fallback;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function writeBoolSetting(key, value) {
|
|||
|
|
try {
|
|||
|
|
localStorage.setItem(key, value ? 'true' : 'false');
|
|||
|
|
} catch {
|
|||
|
|
// Keep the live checkbox state even if storage is unavailable.
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function shouldSendAudioToAi() {
|
|||
|
|
return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function shouldSendVideoToAi() {
|
|||
|
|
return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function appendChatLine(role, text, important = false) {
|
|||
|
|
const box = document.getElementById('native-copilot-log');
|
|||
|
|
if (!box) return;
|
|||
|
|
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
|||
|
|
|
|||
|
|
const row = document.createElement('div');
|
|||
|
|
row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`;
|
|||
|
|
row.style.marginBottom = '4px';
|
|||
|
|
if (important) row.style.color = '#fcd34d';
|
|||
|
|
box.appendChild(row);
|
|||
|
|
while (box.children.length > 120) box.removeChild(box.firstChild);
|
|||
|
|
box.scrollTop = box.scrollHeight;
|
|||
|
|
return row;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function startAssistantLine() {
|
|||
|
|
assistantBuffer = '';
|
|||
|
|
assistantLine = null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function appendAssistantDelta(delta) {
|
|||
|
|
if (!delta) return;
|
|||
|
|
const box = document.getElementById('native-copilot-log');
|
|||
|
|
if (!box) {
|
|||
|
|
console.log(`[AI Native Copilot] AI: ${delta}`);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
|
|||
|
|
if (!assistantLine) assistantLine = appendChatLine('AI', '');
|
|||
|
|
assistantBuffer += delta;
|
|||
|
|
assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`;
|
|||
|
|
box.scrollTop = box.scrollHeight;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function finishAssistantLine() {
|
|||
|
|
assistantLine = null;
|
|||
|
|
assistantBuffer = '';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function logUserTranscript(text) {
|
|||
|
|
if (!text?.trim()) return;
|
|||
|
|
finishAssistantLine();
|
|||
|
|
appendChatLine('用户', text.trim(), true);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function recorderMimeType() {
|
|||
|
|
if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus';
|
|||
|
|
if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm';
|
|||
|
|
return '';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function recordingFileName(track) {
|
|||
|
|
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|||
|
|
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
|||
|
|
return `native-ai-input-${trackPart}-${stamp}.webm`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function videoRecordingFileName(track) {
|
|||
|
|
const stamp = new Date().toISOString().replace(/[:.]/g, '-');
|
|||
|
|
const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
|
|||
|
|
return `native-ai-video-${trackPart}-${stamp}.webm`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function updateRecordButton() {
|
|||
|
|
const button = document.getElementById('native-copilot-record');
|
|||
|
|
if (!button) return;
|
|||
|
|
const recording = inputRecorder?.state === 'recording';
|
|||
|
|
button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入';
|
|||
|
|
button.style.background = recording ? '#dc2626' : '#2563eb';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function videoRecorderMimeType() {
|
|||
|
|
if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8';
|
|||
|
|
if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm';
|
|||
|
|
return '';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function updateVideoRecordButton() {
|
|||
|
|
const button = document.getElementById('native-copilot-record-video');
|
|||
|
|
if (!button) return;
|
|||
|
|
const recording = videoRecorder?.state === 'recording';
|
|||
|
|
button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频';
|
|||
|
|
button.style.background = recording ? '#dc2626' : '#475569';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function startInputRecording() {
|
|||
|
|
if (inputRecorder?.state === 'recording') return;
|
|||
|
|
chooseRemoteAudioTrack('record');
|
|||
|
|
if (!isUsableTrack(remoteUserAudioTrack)) {
|
|||
|
|
appendChatLine('录音', '没有可录制的 AI 输入音轨', true);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
if (!window.MediaRecorder) {
|
|||
|
|
appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
inputRecordingChunks = [];
|
|||
|
|
inputRecordingTrack = remoteUserAudioTrack;
|
|||
|
|
inputRecordingStartedAt = Date.now();
|
|||
|
|
const stream = new MediaStream([inputRecordingTrack]);
|
|||
|
|
const mimeType = recorderMimeType();
|
|||
|
|
inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
|||
|
|
inputRecorder.ondataavailable = (event) => {
|
|||
|
|
if (event.data?.size > 0) inputRecordingChunks.push(event.data);
|
|||
|
|
};
|
|||
|
|
inputRecorder.onstop = () => {
|
|||
|
|
const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' });
|
|||
|
|
const url = URL.createObjectURL(blob);
|
|||
|
|
const link = document.createElement('a');
|
|||
|
|
link.href = url;
|
|||
|
|
link.download = recordingFileName(inputRecordingTrack);
|
|||
|
|
document.body.appendChild(link);
|
|||
|
|
link.click();
|
|||
|
|
link.remove();
|
|||
|
|
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
|||
|
|
|
|||
|
|
const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1);
|
|||
|
|
appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
|||
|
|
inputRecorder = null;
|
|||
|
|
inputRecordingChunks = [];
|
|||
|
|
inputRecordingTrack = null;
|
|||
|
|
updateRecordButton();
|
|||
|
|
};
|
|||
|
|
inputRecorder.start(250);
|
|||
|
|
appendChatLine('录音', `开始录制 AI 输入：${describeTrack(inputRecordingTrack)}`, true);
|
|||
|
|
updateRecordButton();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function stopInputRecording() {
|
|||
|
|
if (inputRecorder?.state === 'recording') inputRecorder.stop();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function toggleInputRecording() {
|
|||
|
|
if (inputRecorder?.state === 'recording') stopInputRecording();
|
|||
|
|
else startInputRecording();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function startVideoRecording() {
|
|||
|
|
if (videoRecorder?.state === 'recording') return;
|
|||
|
|
chooseRemoteVideoTrack('record-video');
|
|||
|
|
if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) {
|
|||
|
|
appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
if (!window.MediaRecorder) {
|
|||
|
|
appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
videoRecordingChunks = [];
|
|||
|
|
videoRecordingTrack = remoteUserVideoTrack;
|
|||
|
|
videoRecordingStartedAt = Date.now();
|
|||
|
|
const stream = new MediaStream([videoRecordingTrack]);
|
|||
|
|
const mimeType = videoRecorderMimeType();
|
|||
|
|
videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
|
|||
|
|
videoRecorder.ondataavailable = (event) => {
|
|||
|
|
if (event.data?.size > 0) videoRecordingChunks.push(event.data);
|
|||
|
|
};
|
|||
|
|
videoRecorder.onstop = () => {
|
|||
|
|
const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' });
|
|||
|
|
const url = URL.createObjectURL(blob);
|
|||
|
|
const link = document.createElement('a');
|
|||
|
|
link.href = url;
|
|||
|
|
link.download = videoRecordingFileName(videoRecordingTrack);
|
|||
|
|
document.body.appendChild(link);
|
|||
|
|
link.click();
|
|||
|
|
link.remove();
|
|||
|
|
setTimeout(() => URL.revokeObjectURL(url), 30000);
|
|||
|
|
|
|||
|
|
const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1);
|
|||
|
|
appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
|
|||
|
|
videoRecorder = null;
|
|||
|
|
videoRecordingChunks = [];
|
|||
|
|
videoRecordingTrack = null;
|
|||
|
|
updateVideoRecordButton();
|
|||
|
|
};
|
|||
|
|
videoRecorder.start(250);
|
|||
|
|
appendChatLine('录制视频', `开始录制 AI 视频：${describeTrack(videoRecordingTrack)}`, true);
|
|||
|
|
updateVideoRecordButton();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function stopVideoRecording() {
|
|||
|
|
if (videoRecorder?.state === 'recording') videoRecorder.stop();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function toggleVideoRecording() {
|
|||
|
|
if (videoRecorder?.state === 'recording') stopVideoRecording();
|
|||
|
|
else startVideoRecording();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function generatePcId() {
|
|||
|
|
const bytes = new Uint8Array(16);
|
|||
|
|
crypto.getRandomValues(bytes);
|
|||
|
|
return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function isInternalPc(pc) {
|
|||
|
|
return Boolean(pc?.__aiNativeCopilotInternal);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function isUsableTrack(track) {
|
|||
|
|
return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function isLiveTrack(track) {
|
|||
|
|
return track && track.readyState === 'live';
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function candidateFor(track, candidates) {
|
|||
|
|
let candidate = candidates.get(track.id);
|
|||
|
|
if (!candidate) {
|
|||
|
|
candidate = {
|
|||
|
|
track,
|
|||
|
|
receiver: null,
|
|||
|
|
reasons: new Set(),
|
|||
|
|
elements: new Set(),
|
|||
|
|
firstSeenAt: Date.now(),
|
|||
|
|
lastSeenAt: 0,
|
|||
|
|
lastPacketAt: 0,
|
|||
|
|
bytesReceived: 0,
|
|||
|
|
packetsReceived: 0,
|
|||
|
|
bytesDelta: 0,
|
|||
|
|
packetsDelta: 0,
|
|||
|
|
level: 0,
|
|||
|
|
score: 0,
|
|||
|
|
logged: false,
|
|||
|
|
};
|
|||
|
|
candidates.set(track.id, candidate);
|
|||
|
|
}
|
|||
|
|
candidate.track = track;
|
|||
|
|
candidate.lastSeenAt = Date.now();
|
|||
|
|
return candidate;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function businessSenderCandidateFor(sender, track, reason) {
|
|||
|
|
if (!sender) return null;
|
|||
|
|
const senderTrack = track || sender.track;
|
|||
|
|
if (senderTrack && senderTrack.kind !== 'audio') return null;
|
|||
|
|
|
|||
|
|
const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`;
|
|||
|
|
sender.__aiNativeCopilotBusinessSenderId = id;
|
|||
|
|
|
|||
|
|
let candidate = businessSenderCandidates.get(id);
|
|||
|
|
if (!candidate) {
|
|||
|
|
candidate = {
|
|||
|
|
id,
|
|||
|
|
sender,
|
|||
|
|
track: senderTrack || null,
|
|||
|
|
originalTrack: senderTrack || null,
|
|||
|
|
sources: new Set(),
|
|||
|
|
bytesSent: 0,
|
|||
|
|
packetsSent: 0,
|
|||
|
|
bytesDelta: 0,
|
|||
|
|
packetsDelta: 0,
|
|||
|
|
score: 0,
|
|||
|
|
logged: false,
|
|||
|
|
};
|
|||
|
|
businessSenderCandidates.set(id, candidate);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
candidate.sender = sender;
|
|||
|
|
candidate.track = senderTrack || sender.track || candidate.track;
|
|||
|
|
if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack) {
|
|||
|
|
candidate.originalTrack = senderTrack;
|
|||
|
|
}
|
|||
|
|
candidate.sources.add(reason);
|
|||
|
|
wrapBusinessSender(sender);
|
|||
|
|
return candidate;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function findReceiverForTrack(track) {
|
|||
|
|
if (!track) return null;
|
|||
|
|
return Array.from(window.__aiNativeCopilotReceivers || [])
|
|||
|
|
.find((receiver) => receiver.track === track) || null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function rememberReceiver(receiver, reason) {
|
|||
|
|
if (!receiver?.track || isInternalPc(receiver.transport)) return;
|
|||
|
|
window.__aiNativeCopilotReceivers.add(receiver);
|
|||
|
|
const track = receiver.track;
|
|||
|
|
if (!isUsableTrack(track)) return;
|
|||
|
|
|
|||
|
|
if (track.kind === 'audio') {
|
|||
|
|
const candidate = candidateFor(track, audioCandidates);
|
|||
|
|
candidate.receiver = receiver;
|
|||
|
|
candidate.reasons.add(reason);
|
|||
|
|
} else if (track.kind === 'video') {
|
|||
|
|
const candidate = candidateFor(track, videoCandidates);
|
|||
|
|
candidate.receiver = receiver;
|
|||
|
|
candidate.reasons.add(reason);
|
|||
|
|
chooseRemoteVideoTrack(reason);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function setVuWidth(id, level) {
|
|||
|
|
const el = document.getElementById(id);
|
|||
|
|
if (!el) return;
|
|||
|
|
el.style.width = `${Math.min(100, level * 2500)}%`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function renderVu() {
|
|||
|
|
inputVuLevel *= 0.88;
|
|||
|
|
aiOutVuLevel *= 0.88;
|
|||
|
|
setVuWidth('native-copilot-input-vu', inputVuLevel);
|
|||
|
|
setVuWidth('native-copilot-ai-vu', aiOutVuLevel);
|
|||
|
|
requestAnimationFrame(renderVu);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function levelFromStats(stat, memory) {
|
|||
|
|
let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0;
|
|||
|
|
if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') {
|
|||
|
|
const prev = memory.get(stat.id);
|
|||
|
|
memory.set(stat.id, {
|
|||
|
|
energy: stat.totalAudioEnergy,
|
|||
|
|
duration: stat.totalSamplesDuration,
|
|||
|
|
});
|
|||
|
|
if (prev) {
|
|||
|
|
const energyDelta = stat.totalAudioEnergy - prev.energy;
|
|||
|
|
const durationDelta = stat.totalSamplesDuration - prev.duration;
|
|||
|
|
if (energyDelta > 0 && durationDelta > 0) {
|
|||
|
|
level = Math.max(level, Math.sqrt(energyDelta / durationDelta));
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return level;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function refreshVuFromStats() {
|
|||
|
|
try {
|
|||
|
|
if (!(await checkLockedUserTracks('stats-before'))) return;
|
|||
|
|
await refreshAudioCandidateStats();
|
|||
|
|
chooseRemoteAudioTrack('stats');
|
|||
|
|
|
|||
|
|
await refreshBusinessSenderStats();
|
|||
|
|
chooseBusinessAudioSender('stats');
|
|||
|
|
await checkLockedUserTracks('stats-after');
|
|||
|
|
} catch (error) {
|
|||
|
|
// Stats APIs are best-effort across browsers.
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function refreshBusinessSenderStats() {
|
|||
|
|
for (const candidate of businessSenderCandidates.values()) {
|
|||
|
|
const sender = candidate.sender;
|
|||
|
|
if (!sender?.getStats) continue;
|
|||
|
|
candidate.track = sender.track || candidate.track;
|
|||
|
|
|
|||
|
|
const report = await sender.getStats();
|
|||
|
|
report.forEach((stat) => {
|
|||
|
|
const kind = stat.kind || stat.mediaType;
|
|||
|
|
if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
|
|||
|
|
|
|||
|
|
const key = `${candidate.id}:${stat.id}`;
|
|||
|
|
const prev = outboundStatsMemory.get(key);
|
|||
|
|
const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0;
|
|||
|
|
const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0;
|
|||
|
|
candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
|||
|
|
candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
|||
|
|
candidate.bytesSent = bytes;
|
|||
|
|
candidate.packetsSent = packets;
|
|||
|
|
outboundStatsMemory.set(key, { bytes, packets });
|
|||
|
|
|
|||
|
|
if (candidate.sender === businessAudioSender) {
|
|||
|
|
businessOutboundStats.bytesDelta = candidate.bytesDelta;
|
|||
|
|
businessOutboundStats.packetsDelta = candidate.packetsDelta;
|
|||
|
|
businessOutboundStats.bytesSent = candidate.bytesSent;
|
|||
|
|
businessOutboundStats.packetsSent = candidate.packetsSent;
|
|||
|
|
aiOutVuLevel = Math.max(
|
|||
|
|
aiOutVuLevel,
|
|||
|
|
levelFromStats(stat, senderStatsMemory),
|
|||
|
|
outputVuLevelFromStats()
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function refreshAudioCandidateStats() {
|
|||
|
|
for (const candidate of audioCandidates.values()) {
|
|||
|
|
if (!isUsableTrack(candidate.track)) continue;
|
|||
|
|
const receiver = candidate.receiver || findReceiverForTrack(candidate.track);
|
|||
|
|
if (!receiver?.getStats) continue;
|
|||
|
|
candidate.receiver = receiver;
|
|||
|
|
|
|||
|
|
const report = await receiver.getStats();
|
|||
|
|
report.forEach((stat) => {
|
|||
|
|
const kind = stat.kind || stat.mediaType;
|
|||
|
|
if (stat.type !== 'inbound-rtp' || kind !== 'audio') return;
|
|||
|
|
|
|||
|
|
const prev = inboundStatsMemory.get(stat.id);
|
|||
|
|
const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0;
|
|||
|
|
const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0;
|
|||
|
|
const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
|
|||
|
|
const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
|
|||
|
|
inboundStatsMemory.set(stat.id, { bytes, packets });
|
|||
|
|
|
|||
|
|
candidate.bytesReceived = bytes;
|
|||
|
|
candidate.packetsReceived = packets;
|
|||
|
|
candidate.bytesDelta = bytesDelta;
|
|||
|
|
candidate.packetsDelta = packetsDelta;
|
|||
|
|
candidate.level = levelFromStats(stat, receiverStatsMemory);
|
|||
|
|
if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) {
|
|||
|
|
candidate.lastPacketAt = Date.now();
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (candidate.track === remoteUserAudioTrack) {
|
|||
|
|
inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate));
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function outboundAudioBytes(sender) {
|
|||
|
|
if (!sender?.getStats) return null;
|
|||
|
|
try {
|
|||
|
|
let total = 0;
|
|||
|
|
const report = await sender.getStats();
|
|||
|
|
report.forEach((stat) => {
|
|||
|
|
const kind = stat.kind || stat.mediaType;
|
|||
|
|
if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') {
|
|||
|
|
total += stat.bytesSent;
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
return total;
|
|||
|
|
} catch {
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function aiReadyForMedia() {
|
|||
|
|
if (!aiPc || !aiPc.remoteDescription) return false;
|
|||
|
|
return (
|
|||
|
|
aiPc.connectionState === 'connected'
|
|||
|
|
|| aiPc.iceConnectionState === 'connected'
|
|||
|
|
|| aiPc.iceConnectionState === 'completed'
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function aiStateLabel() {
|
|||
|
|
if (!aiPc) return 'pc=null';
|
|||
|
|
return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function waitForAiMediaReady(timeoutMs = 10000) {
|
|||
|
|
if (aiReadyForMedia()) return Promise.resolve(true);
|
|||
|
|
return new Promise((resolve) => {
|
|||
|
|
const startedAt = Date.now();
|
|||
|
|
const timer = setInterval(() => {
|
|||
|
|
if (aiReadyForMedia()) {
|
|||
|
|
clearInterval(timer);
|
|||
|
|
resolve(true);
|
|||
|
|
} else if (Date.now() - startedAt >= timeoutMs) {
|
|||
|
|
clearInterval(timer);
|
|||
|
|
resolve(false);
|
|||
|
|
}
|
|||
|
|
}, 250);
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scheduleAiInputHealthCheck(track) {
|
|||
|
|
if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer);
|
|||
|
|
aiInputHealthTimer = setTimeout(async () => {
|
|||
|
|
if (!aiAudioSender || aiAudioSender.track !== track) return;
|
|||
|
|
log(`checking native AI input after media connects: ${aiStateLabel()}`);
|
|||
|
|
const ready = await waitForAiMediaReady(10000);
|
|||
|
|
if (!ready || !aiAudioSender || aiAudioSender.track !== track) {
|
|||
|
|
log(`native AI input could not be verified: ${aiStateLabel()}`);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const before = await outboundAudioBytes(aiAudioSender);
|
|||
|
|
await new Promise((resolve) => setTimeout(resolve, 2500));
|
|||
|
|
if (!aiAudioSender || aiAudioSender.track !== track) return;
|
|||
|
|
const after = await outboundAudioBytes(aiAudioSender);
|
|||
|
|
log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true);
|
|||
|
|
}, 500);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function describeTrack(track) {
|
|||
|
|
if (!track) return 'none';
|
|||
|
|
return `${track.id}${track.muted ? ' muted' : ''}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function elementLayoutInfo(element) {
|
|||
|
|
if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null;
|
|||
|
|
const rect = element.getBoundingClientRect();
|
|||
|
|
const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1;
|
|||
|
|
const centerY = rect.top + rect.height / 2;
|
|||
|
|
const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0));
|
|||
|
|
const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0));
|
|||
|
|
return {
|
|||
|
|
tag: element.tagName,
|
|||
|
|
top: Math.round(rect.top),
|
|||
|
|
left: Math.round(rect.left),
|
|||
|
|
width: Math.round(rect.width),
|
|||
|
|
height: Math.round(rect.height),
|
|||
|
|
centerY: Math.round(centerY),
|
|||
|
|
upperHalf: centerY <= viewportHeight * 0.55,
|
|||
|
|
visibleArea: Math.round(visibleWidth * visibleHeight),
|
|||
|
|
paused: Boolean(element.paused),
|
|||
|
|
muted: Boolean(element.muted),
|
|||
|
|
volume: typeof element.volume === 'number' ? element.volume : null,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function candidateLayoutScore(candidate) {
|
|||
|
|
let score = 0;
|
|||
|
|
for (const element of candidate?.elements || []) {
|
|||
|
|
const info = elementLayoutInfo(element);
|
|||
|
|
if (!info) continue;
|
|||
|
|
if (info.upperHalf) score += 9000;
|
|||
|
|
else score -= 2500;
|
|||
|
|
if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80);
|
|||
|
|
if (info.tag === 'VIDEO') score += 1200;
|
|||
|
|
if (info.tag === 'AUDIO') score += 300;
|
|||
|
|
if (!info.paused) score += 700;
|
|||
|
|
if (!info.muted && (info.volume ?? 1) > 0) score += 500;
|
|||
|
|
}
|
|||
|
|
return score;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function hasUpperHalfElement(candidate) {
|
|||
|
|
return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function lockedTrackUsable(track, candidates) {
|
|||
|
|
const candidate = candidates.get(track?.id);
|
|||
|
|
return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function handleLockedUserTrackLost(kind, reason) {
|
|||
|
|
const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效（${reason}），已断开 AI`;
|
|||
|
|
appendChatLine('选轨', message, true);
|
|||
|
|
log(message);
|
|||
|
|
if (kind === 'audio') {
|
|||
|
|
remoteUserAudioLocked = false;
|
|||
|
|
remoteUserAudioTrack = null;
|
|||
|
|
} else {
|
|||
|
|
remoteUserVideoLocked = false;
|
|||
|
|
remoteUserVideoTrack = null;
|
|||
|
|
}
|
|||
|
|
if (isAiTakeover) {
|
|||
|
|
isAiTakeover = false;
|
|||
|
|
updateModeButton();
|
|||
|
|
await restoreBusinessAudio();
|
|||
|
|
disconnectAi();
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function checkLockedUserTracks(reason) {
|
|||
|
|
if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) {
|
|||
|
|
await handleLockedUserTrackLost('audio', reason);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) {
|
|||
|
|
await handleLockedUserTrackLost('video', reason);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scoreAudioCandidate(candidate) {
|
|||
|
|
if (!candidate || !isUsableTrack(candidate.track)) return -1;
|
|||
|
|
const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000;
|
|||
|
|
const hasAudibleLevel = candidate.level >= 0.0015;
|
|||
|
|
let score = 0;
|
|||
|
|
if (candidate.track.muted) score -= 5000;
|
|||
|
|
else score += 800;
|
|||
|
|
score += candidateLayoutScore(candidate);
|
|||
|
|
if (candidate.receiver) score += 600;
|
|||
|
|
if (candidate.bytesReceived > 0) score += 100;
|
|||
|
|
if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2);
|
|||
|
|
if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8);
|
|||
|
|
if (hasRecentPackets) score += 400;
|
|||
|
|
if (hasAudibleLevel) score += 5000;
|
|||
|
|
score += Math.min(12000, candidate.level * 900000);
|
|||
|
|
if (!hasAudibleLevel && hasRecentPackets) score -= 1800;
|
|||
|
|
if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80;
|
|||
|
|
return score;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scoreVideoCandidate(candidate) {
|
|||
|
|
if (!candidate || !isUsableTrack(candidate.track)) return -1;
|
|||
|
|
let score = 0;
|
|||
|
|
if (candidate.track.muted) score -= 1000;
|
|||
|
|
else score += 500;
|
|||
|
|
if (candidate.receiver) score += 600;
|
|||
|
|
score += candidateLayoutScore(candidate);
|
|||
|
|
for (const element of candidate.elements || []) {
|
|||
|
|
const info = elementLayoutInfo(element);
|
|||
|
|
if (!info) continue;
|
|||
|
|
if (info.tag === 'VIDEO') score += 2500;
|
|||
|
|
if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50);
|
|||
|
|
}
|
|||
|
|
if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100;
|
|||
|
|
return score;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function inputVuLevelFromCandidate(candidate) {
|
|||
|
|
if (!candidate) return 0;
|
|||
|
|
return Math.max(
|
|||
|
|
candidate.level,
|
|||
|
|
Math.min(0.04, candidate.bytesDelta / 16000),
|
|||
|
|
Math.min(0.02, candidate.packetsDelta / 800)
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function updateSelectedInputLabel() {
|
|||
|
|
const label = document.getElementById('native-copilot-selected-input');
|
|||
|
|
if (!label) return;
|
|||
|
|
const candidate = audioCandidates.get(remoteUserAudioTrack?.id);
|
|||
|
|
if (!candidate) {
|
|||
|
|
label.textContent = '等待远端音轨';
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean);
|
|||
|
|
const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout';
|
|||
|
|
label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function outputVuLevelFromStats() {
|
|||
|
|
return Math.max(
|
|||
|
|
Math.min(0.12, businessOutboundStats.bytesDelta / 12000),
|
|||
|
|
Math.min(0.04, businessOutboundStats.packetsDelta / 500)
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function isKnownRemoteAudioTrack(track) {
|
|||
|
|
return Boolean(track && (
|
|||
|
|
track === remoteUserAudioTrack
|
|||
|
|
|| audioCandidates.has(track.id)
|
|||
|
|
));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function isAiOutputTrack(track) {
|
|||
|
|
return Boolean(track && track === aiResponseAudioTrack);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scoreBusinessSenderCandidate(candidate) {
|
|||
|
|
if (!candidate?.sender) return -1;
|
|||
|
|
const track = candidate.sender.track || candidate.track;
|
|||
|
|
let score = 0;
|
|||
|
|
|
|||
|
|
if (track?.kind === 'audio') score += 500;
|
|||
|
|
else if (!track) score -= 1000;
|
|||
|
|
|
|||
|
|
if (track?.readyState === 'live') score += 700;
|
|||
|
|
if (track && !track.muted) score += 300;
|
|||
|
|
if (candidate.sources.has('pc-addTrack')) score += 5000;
|
|||
|
|
if (candidate.sources.has('pc-addTransceiver')) score -= 1000;
|
|||
|
|
if (track && localCaptureTrackIds.has(track.id)) score += 2500;
|
|||
|
|
if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500;
|
|||
|
|
if (isKnownRemoteAudioTrack(track)) score -= 9000;
|
|||
|
|
if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000;
|
|||
|
|
if (isAiOutputTrack(track)) score += 4500;
|
|||
|
|
if (candidate.bytesSent > 0) score += 600;
|
|||
|
|
if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4);
|
|||
|
|
if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10);
|
|||
|
|
|
|||
|
|
return score;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function chooseBusinessAudioSender(reason) {
|
|||
|
|
let best = null;
|
|||
|
|
let bestScore = -1;
|
|||
|
|
for (const candidate of businessSenderCandidates.values()) {
|
|||
|
|
candidate.score = scoreBusinessSenderCandidate(candidate);
|
|||
|
|
if (candidate.score > bestScore) {
|
|||
|
|
best = candidate;
|
|||
|
|
bestScore = candidate.score;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!best || bestScore < 0) return false;
|
|||
|
|
if (businessAudioSender === best.sender) return true;
|
|||
|
|
|
|||
|
|
businessAudioSender = best.sender;
|
|||
|
|
businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack;
|
|||
|
|
log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true);
|
|||
|
|
attachAiOutputToBusiness();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function chooseRemoteAudioTrack(reason) {
|
|||
|
|
const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id);
|
|||
|
|
if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) {
|
|||
|
|
updateSelectedInputLabel();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
if (remoteUserAudioLocked) return false;
|
|||
|
|
|
|||
|
|
let best = null;
|
|||
|
|
let bestScore = -1;
|
|||
|
|
for (const candidate of audioCandidates.values()) {
|
|||
|
|
candidate.score = scoreAudioCandidate(candidate);
|
|||
|
|
if (candidate.score > bestScore) {
|
|||
|
|
best = candidate;
|
|||
|
|
bestScore = candidate.score;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!best || bestScore < 0) return false;
|
|||
|
|
if (remoteUserAudioTrack === best.track) {
|
|||
|
|
updateSelectedInputLabel();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const current = audioCandidates.get(remoteUserAudioTrack?.id);
|
|||
|
|
const currentScore = scoreAudioCandidate(current);
|
|||
|
|
const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted
|
|||
|
|
&& (current.level >= 0.0015 || hasUpperHalfElement(current))
|
|||
|
|
&& Date.now() - current.lastPacketAt < 3000;
|
|||
|
|
if (currentHealthy && bestScore < currentScore + 1800) {
|
|||
|
|
updateSelectedInputLabel();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
remoteUserAudioTrack = best.track;
|
|||
|
|
remoteUserAudioLocked = hasUpperHalfElement(best);
|
|||
|
|
log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true);
|
|||
|
|
updateSelectedInputLabel();
|
|||
|
|
attachRemoteInputToAi();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function chooseRemoteVideoTrack(reason) {
|
|||
|
|
const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id);
|
|||
|
|
if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) {
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
if (remoteUserVideoLocked) return false;
|
|||
|
|
|
|||
|
|
let best = null;
|
|||
|
|
let bestScore = -1;
|
|||
|
|
for (const candidate of videoCandidates.values()) {
|
|||
|
|
candidate.score = scoreVideoCandidate(candidate);
|
|||
|
|
if (candidate.score > bestScore) {
|
|||
|
|
best = candidate;
|
|||
|
|
bestScore = candidate.score;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!best || bestScore < 0) return false;
|
|||
|
|
if (remoteUserVideoTrack === best.track) return true;
|
|||
|
|
|
|||
|
|
const current = videoCandidates.get(remoteUserVideoTrack?.id);
|
|||
|
|
const currentScore = scoreVideoCandidate(current);
|
|||
|
|
if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true;
|
|||
|
|
|
|||
|
|
remoteUserVideoTrack = best.track;
|
|||
|
|
remoteUserVideoLocked = hasUpperHalfElement(best);
|
|||
|
|
log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true);
|
|||
|
|
attachRemoteVideoToAi();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function ensureUserTracksBeforeConnect() {
|
|||
|
|
chooseRemoteAudioTrack('pre-connect');
|
|||
|
|
chooseRemoteVideoTrack('pre-connect');
|
|||
|
|
|
|||
|
|
if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) {
|
|||
|
|
appendChatLine('选轨', '未锁定上半部用户音频轨，暂不连接 AI', true);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) {
|
|||
|
|
appendChatLine('选轨', '未锁定上半部用户视频轨，暂不连接 AI', true);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function rememberRemoteTrack(track, reason, element = null) {
|
|||
|
|
if (!isUsableTrack(track)) return;
|
|||
|
|
|
|||
|
|
if (track.kind === 'audio') {
|
|||
|
|
const candidate = candidateFor(track, audioCandidates);
|
|||
|
|
candidate.receiver = candidate.receiver || findReceiverForTrack(track);
|
|||
|
|
candidate.reasons.add(reason);
|
|||
|
|
if (element) candidate.elements.add(element);
|
|||
|
|
if (!candidate.logged) {
|
|||
|
|
candidate.logged = true;
|
|||
|
|
log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true);
|
|||
|
|
}
|
|||
|
|
chooseRemoteAudioTrack(reason);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (track.kind === 'video') {
|
|||
|
|
const candidate = candidateFor(track, videoCandidates);
|
|||
|
|
candidate.receiver = candidate.receiver || findReceiverForTrack(track);
|
|||
|
|
candidate.reasons.add(reason);
|
|||
|
|
if (element) candidate.elements.add(element);
|
|||
|
|
if (!candidate.logged) {
|
|||
|
|
candidate.logged = true;
|
|||
|
|
log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true);
|
|||
|
|
}
|
|||
|
|
chooseRemoteVideoTrack(reason);
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function rememberRemoteStream(stream, reason, element = null) {
|
|||
|
|
if (!stream || !(stream instanceof MediaStream)) return;
|
|||
|
|
if (stream.__aiNativeCopilotInternal) return;
|
|||
|
|
|
|||
|
|
const liveTracks = stream.getTracks().filter(isUsableTrack);
|
|||
|
|
if (liveTracks.length === 0) return;
|
|||
|
|
|
|||
|
|
remoteStreams.add(stream);
|
|||
|
|
for (const track of liveTracks) rememberRemoteTrack(track, reason, element);
|
|||
|
|
chooseRemoteVideoTrack(reason);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function rememberBusinessAudioSender(sender, track, reason) {
|
|||
|
|
const candidate = businessSenderCandidateFor(sender, track, reason);
|
|||
|
|
if (!candidate) return;
|
|||
|
|
if (!candidate.logged) {
|
|||
|
|
candidate.logged = true;
|
|||
|
|
log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true);
|
|||
|
|
}
|
|||
|
|
chooseBusinessAudioSender(reason);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function wrapBusinessSender(sender) {
|
|||
|
|
if (!sender || sender.__aiNativeCopilotWrapped) return;
|
|||
|
|
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
|
|||
|
|
if (!originalReplaceTrack) return;
|
|||
|
|
|
|||
|
|
sender.replaceTrack = async function (track) {
|
|||
|
|
const result = await originalReplaceTrack(track);
|
|||
|
|
const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack');
|
|||
|
|
if (candidate) {
|
|||
|
|
candidate.track = track || sender.track || candidate.track;
|
|||
|
|
}
|
|||
|
|
if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack) {
|
|||
|
|
if (candidate) candidate.originalTrack = track;
|
|||
|
|
chooseBusinessAudioSender('replaceTrack');
|
|||
|
|
}
|
|||
|
|
return result;
|
|||
|
|
};
|
|||
|
|
sender.__aiNativeCopilotWrapped = true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function attachRemoteInputToAi() {
|
|||
|
|
if (!shouldSendAudioToAi()) return false;
|
|||
|
|
if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false;
|
|||
|
|
if (aiAudioSender.track === remoteUserAudioTrack) return true;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
await aiAudioSender.replaceTrack(remoteUserAudioTrack);
|
|||
|
|
log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true);
|
|||
|
|
scheduleAiInputHealthCheck(remoteUserAudioTrack);
|
|||
|
|
return true;
|
|||
|
|
} catch (error) {
|
|||
|
|
log(`failed to attach native remote audio to AI: ${error.message || error}`);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function attachRemoteVideoToAi() {
|
|||
|
|
if (!shouldSendVideoToAi()) return false;
|
|||
|
|
if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false;
|
|||
|
|
if (aiVideoSender.track === remoteUserVideoTrack) return true;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
await aiVideoSender.replaceTrack(remoteUserVideoTrack);
|
|||
|
|
log('AI video uses native remote video track', true);
|
|||
|
|
return true;
|
|||
|
|
} catch (error) {
|
|||
|
|
log(`failed to attach native remote video to AI: ${error.message || error}`);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function attachAiOutputToBusiness() {
|
|||
|
|
chooseBusinessAudioSender('before-ai-output');
|
|||
|
|
if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) {
|
|||
|
|
log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
if (businessAudioSender.track === aiResponseAudioTrack) return true;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
await businessAudioSender.replaceTrack(aiResponseAudioTrack);
|
|||
|
|
aiTrackAttachedToBusiness = true;
|
|||
|
|
log(`business call now uses native AI response audio track (${aiResponseAudioTrack.id})`, true);
|
|||
|
|
scheduleAiOutputHealthCheck(aiResponseAudioTrack);
|
|||
|
|
return true;
|
|||
|
|
} catch (error) {
|
|||
|
|
aiTrackAttachedToBusiness = false;
|
|||
|
|
log(`failed to attach AI audio to business call: ${error.message || error}`);
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function scheduleAiOutputHealthCheck(track) {
|
|||
|
|
if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer);
|
|||
|
|
aiOutputHealthTimer = setTimeout(async () => {
|
|||
|
|
if (!businessAudioSender || businessAudioSender.track !== track) {
|
|||
|
|
log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`);
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
const before = await outboundAudioBytes(businessAudioSender);
|
|||
|
|
await new Promise((resolve) => setTimeout(resolve, 2500));
|
|||
|
|
if (!businessAudioSender || businessAudioSender.track !== track) return;
|
|||
|
|
const after = await outboundAudioBytes(businessAudioSender);
|
|||
|
|
log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true);
|
|||
|
|
}, 800);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
async function restoreBusinessAudio() {
|
|||
|
|
if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
|
|||
|
|
log('business call restored original audio track', true);
|
|||
|
|
} catch (error) {
|
|||
|
|
log(`failed to restore business audio track: ${error.message || error}`);
|
|||
|
|
} finally {
|
|||
|
|
aiTrackAttachedToBusiness = false;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function wrapTrackListener(listener) {
|
|||
|
|
if (!listener || listener.__aiNativeCopilotWrapped) return listener;
|
|||
|
|
|
|||
|
|
const wrapped = function (event) {
|
|||
|
|
if (!isInternalPc(this) && event?.track) {
|
|||
|
|
rememberReceiver(event.receiver, 'pc-track');
|
|||
|
|
rememberRemoteTrack(event.track, 'pc-track');
|
|||
|
|
for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (typeof listener === 'function') return listener.apply(this, arguments);
|
|||
|
|
if (listener?.handleEvent) return listener.handleEvent(event);
|
|||
|
|
return undefined;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
wrapped.__aiNativeCopilotWrapped = true;
|
|||
|
|
return wrapped;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
RTCPeerConnection.prototype.addEventListener = function (type, listener, options) {
|
|||
|
|
if (type === 'track') {
|
|||
|
|
return originalAddEventListener.call(this, type, wrapTrackListener(listener), options);
|
|||
|
|
}
|
|||
|
|
return originalAddEventListener.call(this, type, listener, options);
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
try {
|
|||
|
|
const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
|
|||
|
|
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
|
|||
|
|
configurable: true,
|
|||
|
|
enumerable: true,
|
|||
|
|
get() {
|
|||
|
|
return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null;
|
|||
|
|
},
|
|||
|
|
set(listener) {
|
|||
|
|
const wrapped = wrapTrackListener(listener);
|
|||
|
|
this.__aiNativeCopilotOnTrack = wrapped;
|
|||
|
|
if (descriptor?.set) descriptor.set.call(this, wrapped);
|
|||
|
|
else this.addEventListener('track', wrapped);
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
} catch (error) {
|
|||
|
|
console.warn('[AI Native Copilot] failed to wrap ontrack', error);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
|
|||
|
|
const sender = originalAddTrack.call(this, track, ...streams);
|
|||
|
|
if (!isInternalPc(this) && track?.kind === 'audio') {
|
|||
|
|
rememberBusinessAudioSender(sender, track, 'pc-addTrack');
|
|||
|
|
}
|
|||
|
|
return sender;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
if (originalAddTransceiver) {
|
|||
|
|
RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
|
|||
|
|
const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
|
|||
|
|
if (!isInternalPc(this)) {
|
|||
|
|
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
|
|||
|
|
if (kind === 'audio') {
|
|||
|
|
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
|
|||
|
|
rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver');
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
return transceiver;
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
RTCPeerConnection.prototype.setRemoteDescription = function () {
|
|||
|
|
const result = originalSetRemoteDescription.apply(this, arguments);
|
|||
|
|
if (!isInternalPc(this)) {
|
|||
|
|
Promise.resolve(result).then(() => {
|
|||
|
|
try {
|
|||
|
|
for (const receiver of this.getReceivers?.() || []) {
|
|||
|
|
rememberReceiver(receiver, 'receiver-after-srd');
|
|||
|
|
rememberRemoteTrack(receiver.track, 'receiver-after-srd');
|
|||
|
|
}
|
|||
|
|
for (const sender of this.getSenders?.() || []) {
|
|||
|
|
rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd');
|
|||
|
|
}
|
|||
|
|
for (const stream of this.getRemoteStreams?.() || []) {
|
|||
|
|
rememberRemoteStream(stream, 'remote-stream-after-srd');
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
console.warn('[AI Native Copilot] failed to scan receivers', error);
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
return result;
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
if (mediaSrcObjectDescriptor?.set) {
|
|||
|
|
Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
|
|||
|
|
configurable: true,
|
|||
|
|
enumerable: mediaSrcObjectDescriptor.enumerable,
|
|||
|
|
get() {
|
|||
|
|
return mediaSrcObjectDescriptor.get.call(this);
|
|||
|
|
},
|
|||
|
|
set(stream) {
|
|||
|
|
if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) {
|
|||
|
|
rememberRemoteStream(stream, 'media-srcObject', this);
|
|||
|
|
}
|
|||
|
|
return mediaSrcObjectDescriptor.set.call(this, stream);
|
|||
|
|
},
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (originalGetUserMedia) {
|
|||
|
|
navigator.mediaDevices.getUserMedia = async function (constraints) {
|
|||
|
|
const stream = await originalGetUserMedia(constraints);
|
|||
|
|
for (const track of stream.getAudioTracks()) {
|
|||
|
|
localCaptureTrackIds.add(track.id);
|
|||
|
|
}
|
|||
|
|
chooseBusinessAudioSender('getUserMedia');
|
|||
|
|
renderPanelSoon();
|
|||
|
|
return stream;
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
setInterval(async () => {
|
|||
|
|
if (!(await checkLockedUserTracks('scan'))) return;
|
|||
|
|
for (const element of document.querySelectorAll('audio, video')) {
|
|||
|
|
if (element.__aiNativeCopilotElement) continue;
|
|||
|
|
rememberRemoteStream(element.srcObject, 'media-scan', element);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
for (const stream of Array.from(remoteStreams)) {
|
|||
|
|
rememberRemoteStream(stream, 'stream-rescan');
|
|||
|
|
}
|
|||
|
|
await checkLockedUserTracks('scan-after');
|
|||
|
|
}, 1000);
|
|||
|
|
|
|||
|
|
window.__aiNativeCopilotDump = function () {
|
|||
|
|
const audio = Array.from(audioCandidates.values()).map((candidate) => ({
|
|||
|
|
id: candidate.track.id,
|
|||
|
|
label: candidate.track.label,
|
|||
|
|
selected: candidate.track === remoteUserAudioTrack,
|
|||
|
|
enabled: candidate.track.enabled,
|
|||
|
|
muted: candidate.track.muted,
|
|||
|
|
readyState: candidate.track.readyState,
|
|||
|
|
hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
|
|||
|
|
bytesReceived: candidate.bytesReceived,
|
|||
|
|
packetsReceived: candidate.packetsReceived,
|
|||
|
|
bytesDelta: candidate.bytesDelta,
|
|||
|
|
packetsDelta: candidate.packetsDelta,
|
|||
|
|
level: candidate.level,
|
|||
|
|
audible: candidate.level >= 0.0015,
|
|||
|
|
layoutScore: Math.round(candidateLayoutScore(candidate)),
|
|||
|
|
elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
|
|||
|
|
vuLevel: inputVuLevelFromCandidate(candidate),
|
|||
|
|
score: Math.round(scoreAudioCandidate(candidate)),
|
|||
|
|
reasons: Array.from(candidate.reasons),
|
|||
|
|
}));
|
|||
|
|
const video = Array.from(videoCandidates.values()).map((candidate) => ({
|
|||
|
|
id: candidate.track.id,
|
|||
|
|
label: candidate.track.label,
|
|||
|
|
selected: candidate.track === remoteUserVideoTrack,
|
|||
|
|
muted: candidate.track.muted,
|
|||
|
|
readyState: candidate.track.readyState,
|
|||
|
|
hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
|
|||
|
|
layoutScore: Math.round(candidateLayoutScore(candidate)),
|
|||
|
|
score: Math.round(scoreVideoCandidate(candidate)),
|
|||
|
|
elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
|
|||
|
|
reasons: Array.from(candidate.reasons),
|
|||
|
|
}));
|
|||
|
|
const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({
|
|||
|
|
id: candidate.id,
|
|||
|
|
selected: candidate.sender === businessAudioSender,
|
|||
|
|
track: describeTrack(candidate.sender.track || candidate.track),
|
|||
|
|
originalTrack: describeTrack(candidate.originalTrack),
|
|||
|
|
isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track),
|
|||
|
|
isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)),
|
|||
|
|
bytesSent: candidate.bytesSent,
|
|||
|
|
packetsSent: candidate.packetsSent,
|
|||
|
|
bytesDelta: candidate.bytesDelta,
|
|||
|
|
packetsDelta: candidate.packetsDelta,
|
|||
|
|
score: Math.round(scoreBusinessSenderCandidate(candidate)),
|
|||
|
|
sources: Array.from(candidate.sources),
|
|||
|
|
}));
|
|||
|
|
return {
|
|||
|
|
selectedAudio: describeTrack(remoteUserAudioTrack),
|
|||
|
|
selectedVideo: describeTrack(remoteUserVideoTrack),
|
|||
|
|
audioLocked: remoteUserAudioLocked,
|
|||
|
|
videoLocked: remoteUserVideoLocked,
|
|||
|
|
aiInputTrack: describeTrack(aiAudioSender?.track),
|
|||
|
|
aiOutputTrack: describeTrack(aiResponseAudioTrack),
|
|||
|
|
businessSenderTrack: describeTrack(businessAudioSender?.track),
|
|||
|
|
aiTrackAttachedToBusiness,
|
|||
|
|
videoRecording: videoRecorder?.state === 'recording' ? {
|
|||
|
|
track: describeTrack(videoRecordingTrack),
|
|||
|
|
seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1),
|
|||
|
|
chunks: videoRecordingChunks.length,
|
|||
|
|
} : null,
|
|||
|
|
businessOutboundStats: { ...businessOutboundStats },
|
|||
|
|
aiState: aiStateLabel(),
|
|||
|
|
audio,
|
|||
|
|
video,
|
|||
|
|
businessSenders,
|
|||
|
|
};
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
async function connectAi(signalingUrl, assistantId) {
|
|||
|
|
if (!ensureUserTracksBeforeConnect()) {
|
|||
|
|
setStatus('disconnected');
|
|||
|
|
return false;
|
|||
|
|
}
|
|||
|
|
disconnectAi();
|
|||
|
|
|
|||
|
|
const pcId = generatePcId();
|
|||
|
|
aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
|
|||
|
|
aiPc.__aiNativeCopilotInternal = true;
|
|||
|
|
aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`);
|
|||
|
|
aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`);
|
|||
|
|
|
|||
|
|
aiDc = aiPc.createDataChannel('chat');
|
|||
|
|
aiDc.onopen = () => {
|
|||
|
|
setStatus('connected');
|
|||
|
|
log('AI data channel open', true);
|
|||
|
|
aiDc.send(JSON.stringify({ type: 'client-ready' }));
|
|||
|
|
};
|
|||
|
|
aiDc.onclose = () => setStatus('disconnected');
|
|||
|
|
aiDc.onmessage = (event) => {
|
|||
|
|
try {
|
|||
|
|
const msg = JSON.parse(event.data);
|
|||
|
|
const content = msg.delta || msg.content;
|
|||
|
|
if (msg.type === 'transcript') {
|
|||
|
|
logUserTranscript(content);
|
|||
|
|
} else if (msg.type === 'assistant-text-start') {
|
|||
|
|
startAssistantLine();
|
|||
|
|
} else if (msg.type === 'assistant-text-delta') {
|
|||
|
|
appendAssistantDelta(content);
|
|||
|
|
} else if (msg.type === 'assistant-text-end') {
|
|||
|
|
finishAssistantLine();
|
|||
|
|
} else {
|
|||
|
|
log(`data channel message: ${msg.type}`);
|
|||
|
|
}
|
|||
|
|
} catch {
|
|||
|
|
appendChatLine('DATA', event.data);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
const sendAudio = shouldSendAudioToAi();
|
|||
|
|
const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack)
|
|||
|
|
? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' })
|
|||
|
|
: aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' });
|
|||
|
|
aiAudioSender = audioTransceiver.sender;
|
|||
|
|
log(!sendAudio
|
|||
|
|
? 'AI audio uplink disabled by panel setting'
|
|||
|
|
: isUsableTrack(remoteUserAudioTrack)
|
|||
|
|
? 'AI input starts with native remote audio track'
|
|||
|
|
: 'AI input waiting for native remote audio track', true);
|
|||
|
|
|
|||
|
|
const sendVideo = shouldSendVideoToAi();
|
|||
|
|
if (sendVideo) {
|
|||
|
|
const videoTransceiver = isUsableTrack(remoteUserVideoTrack)
|
|||
|
|
? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' })
|
|||
|
|
: aiPc.addTransceiver('video', { direction: 'sendonly' });
|
|||
|
|
aiVideoSender = videoTransceiver.sender;
|
|||
|
|
if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true);
|
|||
|
|
} else {
|
|||
|
|
aiVideoSender = null;
|
|||
|
|
log('AI video uplink disabled by panel setting');
|
|||
|
|
}
|
|||
|
|
if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack);
|
|||
|
|
|
|||
|
|
aiPc.ontrack = (event) => {
|
|||
|
|
if (event.track.kind !== 'audio') return;
|
|||
|
|
ignoredTrackIds.add(event.track.id);
|
|||
|
|
aiResponseAudioTrack = event.track;
|
|||
|
|
log('received native AI response audio track', true);
|
|||
|
|
attachAiOutputToBusiness();
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
aiPc.onicecandidate = (event) => {
|
|||
|
|
if (aiWs?.readyState !== WebSocket.OPEN) return;
|
|||
|
|
aiWs.send(JSON.stringify({
|
|||
|
|
type: 'ice-candidate',
|
|||
|
|
payload: {
|
|||
|
|
pc_id: pcId,
|
|||
|
|
candidate: event.candidate ? {
|
|||
|
|
candidate: event.candidate.candidate,
|
|||
|
|
sdpMid: event.candidate.sdpMid,
|
|||
|
|
sdpMLineIndex: event.candidate.sdpMLineIndex,
|
|||
|
|
} : null,
|
|||
|
|
},
|
|||
|
|
}));
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
setStatus('connecting');
|
|||
|
|
aiWs = new WebSocket(signalingUrl);
|
|||
|
|
aiWs.onopen = async () => {
|
|||
|
|
const offer = await aiPc.createOffer();
|
|||
|
|
await aiPc.setLocalDescription(offer);
|
|||
|
|
aiWs.send(JSON.stringify({
|
|||
|
|
type: 'offer',
|
|||
|
|
payload: {
|
|||
|
|
pc_id: pcId,
|
|||
|
|
sdp: aiPc.localDescription.sdp,
|
|||
|
|
type: aiPc.localDescription.type,
|
|||
|
|
assistant_id: assistantId,
|
|||
|
|
vision_enabled: sendVideo,
|
|||
|
|
},
|
|||
|
|
}));
|
|||
|
|
log('sent AI WebRTC offer', true);
|
|||
|
|
};
|
|||
|
|
aiWs.onmessage = async (event) => {
|
|||
|
|
try {
|
|||
|
|
const msg = JSON.parse(event.data);
|
|||
|
|
if (msg.type === 'answer') {
|
|||
|
|
await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp });
|
|||
|
|
log(`AI WebRTC answer applied; ${aiStateLabel()}`, true);
|
|||
|
|
} else if (msg.type === 'ice-candidate' && msg.payload?.candidate) {
|
|||
|
|
await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {});
|
|||
|
|
} else if (msg.type === 'error') {
|
|||
|
|
throw new Error(msg.payload?.message || 'server error');
|
|||
|
|
}
|
|||
|
|
} catch (error) {
|
|||
|
|
log(`signaling error: ${error.message || error}`);
|
|||
|
|
}
|
|||
|
|
};
|
|||
|
|
aiWs.onerror = () => {
|
|||
|
|
setStatus('failed');
|
|||
|
|
log('AI signaling websocket error');
|
|||
|
|
};
|
|||
|
|
aiWs.onclose = () => setStatus('disconnected');
|
|||
|
|
|
|||
|
|
attachRemoteInputToAi();
|
|||
|
|
if (sendVideo) attachRemoteVideoToAi();
|
|||
|
|
return true;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function disconnectAi() {
|
|||
|
|
if (aiDc) {
|
|||
|
|
try { aiDc.close(); } catch {}
|
|||
|
|
aiDc = null;
|
|||
|
|
}
|
|||
|
|
if (aiPc) {
|
|||
|
|
try { aiPc.close(); } catch {}
|
|||
|
|
aiPc = null;
|
|||
|
|
}
|
|||
|
|
if (aiWs) {
|
|||
|
|
try { aiWs.close(); } catch {}
|
|||
|
|
aiWs = null;
|
|||
|
|
}
|
|||
|
|
aiAudioSender = null;
|
|||
|
|
aiVideoSender = null;
|
|||
|
|
aiResponseAudioTrack = null;
|
|||
|
|
if (inputRecorder?.state === 'recording') stopInputRecording();
|
|||
|
|
if (videoRecorder?.state === 'recording') stopVideoRecording();
|
|||
|
|
setStatus('disconnected');
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function setStatus(status) {
|
|||
|
|
const el = document.getElementById('native-copilot-status');
|
|||
|
|
if (!el) return;
|
|||
|
|
const labels = {
|
|||
|
|
disconnected: '未连接',
|
|||
|
|
connecting: '连接中',
|
|||
|
|
connected: '已连接',
|
|||
|
|
failed: '连接失败',
|
|||
|
|
};
|
|||
|
|
el.textContent = labels[status] || status;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function updateModeButton() {
|
|||
|
|
const button = document.getElementById('native-copilot-toggle');
|
|||
|
|
if (!button) return;
|
|||
|
|
if (isAiTakeover) {
|
|||
|
|
button.textContent = '当前：AI 接管｜点击切回人工';
|
|||
|
|
button.style.background = '#8b5cf6';
|
|||
|
|
} else {
|
|||
|
|
button.textContent = '当前：人工接管｜点击切换 AI';
|
|||
|
|
button.style.background = '#10b981';
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function renderPanel() {
|
|||
|
|
if (!document.body || document.getElementById('native-copilot-panel')) return;
|
|||
|
|
|
|||
|
|
const panel = document.createElement('div');
|
|||
|
|
panel.id = 'native-copilot-panel';
|
|||
|
|
panel.style.cssText = [
|
|||
|
|
'position:fixed',
|
|||
|
|
'top:20px',
|
|||
|
|
'left:20px',
|
|||
|
|
'width:360px',
|
|||
|
|
'z-index:999999',
|
|||
|
|
'box-sizing:border-box',
|
|||
|
|
'padding:16px',
|
|||
|
|
'border-radius:10px',
|
|||
|
|
'background:rgba(17,24,39,.96)',
|
|||
|
|
'color:white',
|
|||
|
|
'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif',
|
|||
|
|
'box-shadow:0 12px 32px rgba(0,0,0,.35)',
|
|||
|
|
].join(';');
|
|||
|
|
|
|||
|
|
panel.innerHTML = `
|
|||
|
|
<div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
|
|||
|
|
<strong style="font-size:15px;color:#93c5fd;">AI 原生选轨</strong>
|
|||
|
|
<span id="native-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
|
|||
|
|
</div>
|
|||
|
|
<div style="font-size:11px;color:#9ca3af;margin-bottom:10px;line-height:1.5;">
|
|||
|
|
音频输入：布局优先自动选轨<br>
|
|||
|
|
视频输入：远端视频轨
|
|||
|
|
</div>
|
|||
|
|
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">信令地址</label>
|
|||
|
|
<input id="native-copilot-url" value="${DEFAULT_SIGNALING_URL}" style="width:100%;box-sizing:border-box;margin-bottom:10px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
|||
|
|
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">助手 ID</label>
|
|||
|
|
<input id="native-copilot-assistant" placeholder="请输入助手 ID" style="width:100%;box-sizing:border-box;margin-bottom:12px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
|
|||
|
|
<div style="display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-bottom:12px;">
|
|||
|
|
<label style="display:flex;align-items:center;gap:6px;padding:8px;border:1px solid #374151;border-radius:6px;background:#111827;font-size:12px;color:#e5e7eb;cursor:pointer;">
|
|||
|
|
<input id="native-copilot-send-audio" type="checkbox" style="margin:0;">
|
|||
|
|
<span>传音频给 AI</span>
|
|||
|
|
</label>
|
|||
|
|
<label style="display:flex;align-items:center;gap:6px;padding:8px;border:1px solid #374151;border-radius:6px;background:#111827;font-size:12px;color:#e5e7eb;cursor:pointer;">
|
|||
|
|
<input id="native-copilot-send-video" type="checkbox" style="margin:0;">
|
|||
|
|
<span>传视频给 AI</span>
|
|||
|
|
</label>
|
|||
|
|
</div>
|
|||
|
|
<div style="margin-bottom:10px;">
|
|||
|
|
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;">
|
|||
|
|
<span id="native-copilot-selected-input" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">等待远端音轨</span><span>原生音轨</span>
|
|||
|
|
</div>
|
|||
|
|
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;">
|
|||
|
|
<div id="native-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div>
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
<div style="margin-bottom:12px;">
|
|||
|
|
<div style="display:flex;justify-content:space-between;font-size:12px;color:#bfdbfe;margin-bottom:4px;">
|
|||
|
|
<span>AI 输出</span><span>业务上行</span>
|
|||
|
|
</div>
|
|||
|
|
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;">
|
|||
|
|
<div id="native-copilot-ai-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div>
|
|||
|
|
</div>
|
|||
|
|
</div>
|
|||
|
|
<button id="native-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前：人工接管｜点击切换 AI</button>
|
|||
|
|
<div style="display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-top:8px;">
|
|||
|
|
<button id="native-copilot-record" style="width:100%;padding:10px;border:0;border-radius:6px;background:#2563eb;color:white;font-weight:700;cursor:pointer;">录制 AI 输入</button>
|
|||
|
|
<button id="native-copilot-record-video" style="width:100%;padding:10px;border:0;border-radius:6px;background:#475569;color:white;font-weight:700;cursor:pointer;">录制 AI 视频</button>
|
|||
|
|
</div>
|
|||
|
|
<div id="native-copilot-log" style="height:170px;margin-top:12px;padding:8px;overflow:auto;border:1px solid #374151;border-radius:6px;background:#030712;color:#a7f3d0;font:11px ui-monospace,Menlo,monospace;">
|
|||
|
|
<span data-placeholder="1" style="color:#6b7280;">等待交互...</span>
|
|||
|
|
</div>
|
|||
|
|
`;
|
|||
|
|
|
|||
|
|
document.body.appendChild(panel);
|
|||
|
|
|
|||
|
|
const toggle = document.getElementById('native-copilot-toggle');
|
|||
|
|
const recordButton = document.getElementById('native-copilot-record');
|
|||
|
|
const videoRecordButton = document.getElementById('native-copilot-record-video');
|
|||
|
|
const urlInput = document.getElementById('native-copilot-url');
|
|||
|
|
const assistantInput = document.getElementById('native-copilot-assistant');
|
|||
|
|
const sendAudioInput = document.getElementById('native-copilot-send-audio');
|
|||
|
|
const sendVideoInput = document.getElementById('native-copilot-send-video');
|
|||
|
|
|
|||
|
|
sendAudioInput.checked = shouldSendAudioToAi();
|
|||
|
|
sendVideoInput.checked = shouldSendVideoToAi();
|
|||
|
|
sendAudioInput.addEventListener('change', () => {
|
|||
|
|
writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked);
|
|||
|
|
log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`);
|
|||
|
|
});
|
|||
|
|
sendVideoInput.addEventListener('change', () => {
|
|||
|
|
writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked);
|
|||
|
|
log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`);
|
|||
|
|
});
|
|||
|
|
recordButton.addEventListener('click', toggleInputRecording);
|
|||
|
|
videoRecordButton.addEventListener('click', toggleVideoRecording);
|
|||
|
|
updateRecordButton();
|
|||
|
|
updateVideoRecordButton();
|
|||
|
|
updateSelectedInputLabel();
|
|||
|
|
updateModeButton();
|
|||
|
|
|
|||
|
|
toggle.addEventListener('click', async () => {
|
|||
|
|
if (!isAiTakeover && !assistantInput.value.trim()) {
|
|||
|
|
alert('请先填写助手 ID。');
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (!isAiTakeover && !ensureUserTracksBeforeConnect()) {
|
|||
|
|
updateModeButton();
|
|||
|
|
return;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
isAiTakeover = !isAiTakeover;
|
|||
|
|
updateModeButton();
|
|||
|
|
if (isAiTakeover) {
|
|||
|
|
const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim());
|
|||
|
|
if (connected) await attachAiOutputToBusiness();
|
|||
|
|
else {
|
|||
|
|
isAiTakeover = false;
|
|||
|
|
updateModeButton();
|
|||
|
|
}
|
|||
|
|
} else {
|
|||
|
|
await restoreBusinessAudio();
|
|||
|
|
disconnectAi();
|
|||
|
|
}
|
|||
|
|
});
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function renderPanelSoon() {
|
|||
|
|
try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if (document.readyState === 'loading') {
|
|||
|
|
document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
|
|||
|
|
} else {
|
|||
|
|
renderPanelSoon();
|
|||
|
|
}
|
|||
|
|
requestAnimationFrame(renderVu);
|
|||
|
|
setInterval(refreshVuFromStats, 250);
|
|||
|
|
setTimeout(renderPanelSoon, 1500);
|
|||
|
|
setTimeout(renderPanelSoon, 3000);
|
|||
|
|
})();
|