2026-07-09 08:41:57 +08:00
// ==UserScript==
// @name 12345 AI Element2 Gated Copilot
// @namespace http://tampermonkey.net/
// @version 1.0
2026-07-09 08:54:15 +08:00
// @description Two-party gated bridge: only feed participant audio into AI after both sides are connected.
2026-07-09 08:41:57 +08:00
// @author You
// @match *://yhy-yw.22sj.cn:*/*
// @grant none
// @run-at document-start
// ==/UserScript==
( function () {
'use strict' ;
2026-07-13 12:12:04 +08:00
const DEFAULT_API_BASE_URL = 'http://182.92.86.220:8000' ;
2026-07-09 08:41:57 +08:00
const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio' ;
const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video' ;
2026-07-09 08:54:15 +08:00
const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/ ;
2026-07-09 08:41:57 +08:00
const originalAddEventListener = RTCPeerConnection . prototype . addEventListener ;
const originalAddTrack = RTCPeerConnection . prototype . addTrack ;
const originalAddTransceiver = RTCPeerConnection . prototype . addTransceiver ;
const originalSetRemoteDescription = RTCPeerConnection . prototype . setRemoteDescription ;
const mediaSrcObjectDescriptor = Object . getOwnPropertyDescriptor ( HTMLMediaElement . prototype , 'srcObject' );
const remoteAudio = new Map ();
const remoteVideo = new Map ();
const businessSenders = new Map ();
const aiOutputTrackIds = new Set ();
const inboundMemory = new Map ();
const outboundMemory = new Map ();
const internalPcs = new WeakSet ();
const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio' ;
const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video' ;
2026-07-13 12:12:04 +08:00
const AUDIO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-audio-selection-mode' ;
const VIDEO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-video-selection-mode' ;
const SENDER_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-sender-selection-mode' ;
2026-07-09 08:41:57 +08:00
let selectedRemoteAudio = null ;
2026-07-13 12:12:04 +08:00
let audioSelectionMode = readAudioSelectionMode ();
let manualAudioTrackId = null ;
let audioTrackSelectSignature = '' ;
2026-07-09 08:41:57 +08:00
let selectedRemoteVideo = null ;
2026-07-13 12:12:04 +08:00
let videoSelectionMode = readVideoSelectionMode ();
let manualVideoTrackId = null ;
let videoTrackSelectSignature = '' ;
2026-07-09 08:41:57 +08:00
let selectedBusinessSender = null ;
2026-07-13 12:12:04 +08:00
let senderSelectionMode = readSenderSelectionMode ();
let manualSenderId = null ;
let senderSelectSignature = '' ;
let changingBusinessSender = false ;
2026-07-09 08:41:57 +08:00
let originalBusinessTrack = null ;
let aiAudioTrack = null ;
let aiPc = null ;
let aiDc = null ;
let aiAudioSender = null ;
let aiVideoSender = null ;
2026-07-13 12:12:04 +08:00
let aiPcId = null ;
let aiOfferUrl = null ;
let aiAssistantId = null ;
let aiAuthorization = null ;
let aiCandidateQueue = [];
let aiCandidateFlushTimer = null ;
let aiKeepAliveTimer = null ;
let aiNegotiating = false ;
let aiCanSendCandidates = false ;
2026-07-09 08:41:57 +08:00
let isAiTakeover = false ;
let aiAttached = false ;
2026-07-09 08:54:15 +08:00
let aiOutputAudioContext = null ;
let aiOutputAnalyser = null ;
let aiOutputAnalyserData = null ;
let aiOutputAnalyserSource = null ;
let aiOutputAnalyserTrack = null ;
2026-07-09 08:41:57 +08:00
let inputVu = 0 ;
let outputVu = 0 ;
let inputHealthTimer = null ;
let outputHealthTimer = null ;
let assistantLine = null ;
let assistantBuffer = '' ;
let inputRecorder = null ;
let inputRecordingChunks = [];
let inputRecordingTrack = null ;
let inputRecordingStartedAt = 0 ;
2026-07-13 12:12:04 +08:00
let senderRecorder = null ;
let senderRecordingChunks = [];
let senderRecordingTrack = null ;
let senderRecordingStartedAt = 0 ;
2026-07-09 08:41:57 +08:00
let videoRecorder = null ;
let videoRecordingChunks = [];
let videoRecordingTrack = null ;
let videoRecordingStartedAt = 0 ;
function log ( message ) {
console . log ( `[AI Element2 Gated] ${ message } ` );
}
function readBoolSetting ( key , fallback ) {
try {
const value = localStorage . getItem ( key );
if ( value === null ) return fallback ;
return value === 'true' ;
} catch {
return fallback ;
}
}
function writeBoolSetting ( key , value ) {
try {
localStorage . setItem ( key , value ? 'true' : 'false' );
} catch {
// Ignore storage failures; the current checkbox state still applies.
}
}
2026-07-13 12:12:04 +08:00
function readAudioSelectionMode () {
try {
if ( ! shouldSendAudioToAi ()) return 'off' ;
return localStorage . getItem ( AUDIO_SELECTION_MODE_STORAGE_KEY ) === 'manual' ? 'manual' : 'auto' ;
} catch {
return 'auto' ;
}
}
function writeAudioSelectionMode ( mode ) {
audioSelectionMode = [ 'auto' , 'manual' , 'off' ]. includes ( mode ) ? mode : 'auto' ;
writeBoolSetting ( AUDIO_UPLINK_STORAGE_KEY , audioSelectionMode !== 'off' );
try {
localStorage . setItem ( AUDIO_SELECTION_MODE_STORAGE_KEY , audioSelectionMode );
} catch {}
}
function readVideoSelectionMode () {
try {
if ( ! shouldSendVideoToAi ()) return 'off' ;
return localStorage . getItem ( VIDEO_SELECTION_MODE_STORAGE_KEY ) === 'manual' ? 'manual' : 'auto' ;
} catch {
return 'auto' ;
}
}
function writeVideoSelectionMode ( mode ) {
videoSelectionMode = [ 'auto' , 'manual' , 'off' ]. includes ( mode ) ? mode : 'auto' ;
writeBoolSetting ( VIDEO_UPLINK_STORAGE_KEY , videoSelectionMode !== 'off' );
try {
localStorage . setItem ( VIDEO_SELECTION_MODE_STORAGE_KEY , videoSelectionMode );
} catch {}
}
function readSenderSelectionMode () {
try {
return localStorage . getItem ( SENDER_SELECTION_MODE_STORAGE_KEY ) === 'manual' ? 'manual' : 'auto' ;
} catch {
return 'auto' ;
}
}
function writeSenderSelectionMode ( mode ) {
senderSelectionMode = mode === 'manual' ? 'manual' : 'auto' ;
try {
localStorage . setItem ( SENDER_SELECTION_MODE_STORAGE_KEY , senderSelectionMode );
} catch {}
}
2026-07-09 08:41:57 +08:00
function shouldSendAudioToAi () {
return readBoolSetting ( AUDIO_UPLINK_STORAGE_KEY , true );
}
function shouldSendVideoToAi () {
return readBoolSetting ( VIDEO_UPLINK_STORAGE_KEY , true );
}
function appendChatLine ( role , text , important = false ) {
const box = document . getElementById ( 'element2-copilot-log' );
if ( ! box ) return ;
if ( box . children . length === 1 && box . children [ 0 ]. dataset . placeholder ) box . innerHTML = '' ;
const row = document . createElement ( 'div' );
row . textContent = `[ ${ new Date (). toLocaleTimeString ([], { hour12 : false } )}] ${ role } : ${ text } ` ;
row . style . marginBottom = '4px' ;
if ( important ) row . style . color = '#fcd34d' ;
box . appendChild ( row );
while ( box . children . length > 120 ) box . removeChild ( box . firstChild );
box . scrollTop = box . scrollHeight ;
return row ;
}
function startAssistantLine () {
assistantBuffer = '' ;
assistantLine = null ;
}
function appendAssistantDelta ( delta ) {
if ( ! delta ) return ;
const box = document . getElementById ( 'element2-copilot-log' );
if ( ! box ) {
console . log ( `[AI Element2 Gated] AI: ${ delta } ` );
return ;
}
if ( box . children . length === 1 && box . children [ 0 ]. dataset . placeholder ) box . innerHTML = '' ;
if ( ! assistantLine ) assistantLine = appendChatLine ( 'AI' , '' );
assistantBuffer += delta ;
assistantLine . textContent = `[ ${ new Date (). toLocaleTimeString ([], { hour12 : false } )}] AI: ${ assistantBuffer } ` ;
box . scrollTop = box . scrollHeight ;
}
function finishAssistantLine () {
assistantLine = null ;
assistantBuffer = '' ;
}
function logUserTranscript ( text ) {
if ( ! text ? . trim ()) return ;
finishAssistantLine ();
appendChatLine ( '用户' , text . trim (), true );
}
function recorderMimeType () {
if ( window . MediaRecorder ? . isTypeSupported ? .( 'audio/webm;codecs=opus' )) return 'audio/webm;codecs=opus' ;
if ( window . MediaRecorder ? . isTypeSupported ? .( 'audio/webm' )) return 'audio/webm' ;
return '' ;
}
function recordingFileName ( track ) {
const stamp = new Date (). toISOString (). replace ( /[:.]/g , '-' );
const trackPart = track ? . id ? track . id . slice ( 0 , 8 ) : 'no-track' ;
return `ai-input- ${ trackPart } - ${ stamp } .webm` ;
}
2026-07-13 12:12:04 +08:00
function senderRecordingFileName ( track ) {
const stamp = new Date (). toISOString (). replace ( /[:.]/g , '-' );
const trackPart = track ? . id ? track . id . slice ( 0 , 8 ) : 'no-track' ;
return `business-sender- ${ trackPart } - ${ stamp } .webm` ;
}
2026-07-09 08:41:57 +08:00
function videoRecordingFileName ( track ) {
const stamp = new Date (). toISOString (). replace ( /[:.]/g , '-' );
const trackPart = track ? . id ? track . id . slice ( 0 , 8 ) : 'no-track' ;
return `ai-video- ${ trackPart } - ${ stamp } .webm` ;
}
function videoRecorderMimeType () {
if ( window . MediaRecorder ? . isTypeSupported ? .( 'video/webm;codecs=vp8' )) return 'video/webm;codecs=vp8' ;
if ( window . MediaRecorder ? . isTypeSupported ? .( 'video/webm' )) return 'video/webm' ;
return '' ;
}
async function startInputRecording () {
if ( inputRecorder ? . state === 'recording' ) return ;
selectRemoteAudio ( 'record' );
if ( ! isLive ( selectedRemoteAudio )) {
appendChatLine ( '录音' , `没有可录制的 ${ STATIC_AUDIO_ELEMENT_REASON } 音轨` , true );
return ;
}
if ( ! window . MediaRecorder ) {
appendChatLine ( '录音' , '当前浏览器不支持 MediaRecorder' , true );
return ;
}
inputRecordingChunks = [];
inputRecordingTrack = selectedRemoteAudio ;
inputRecordingStartedAt = Date . now ();
const stream = new MediaStream ([ selectedRemoteAudio ]);
const mimeType = recorderMimeType ();
inputRecorder = new MediaRecorder ( stream , mimeType ? { mimeType } : undefined );
inputRecorder . ondataavailable = ( event ) => {
if ( event . data ? . size > 0 ) inputRecordingChunks . push ( event . data );
};
inputRecorder . onstop = () => {
const blob = new Blob ( inputRecordingChunks , { type : inputRecorder ? . mimeType || 'audio/webm' });
const url = URL . createObjectURL ( blob );
const link = document . createElement ( 'a' );
link . href = url ;
link . download = recordingFileName ( inputRecordingTrack );
document . body . appendChild ( link );
link . click ();
link . remove ();
setTimeout (() => URL . revokeObjectURL ( url ), 30000 );
const seconds = (( Date . now () - inputRecordingStartedAt ) / 1000 ). toFixed ( 1 );
appendChatLine ( '录音' , `已保存 ${ link . download } ( ${ seconds } s, ${ ( blob . size / 1024 ). toFixed ( 1 ) } KB) ` , true );
inputRecorder = null ;
inputRecordingChunks = [];
inputRecordingTrack = null ;
updateRecordButton ();
};
inputRecorder . start ( 250 );
appendChatLine ( '录音' , `开始录制 AI 输入: ${ trackLabel ( inputRecordingTrack ) } ` , true );
updateRecordButton ();
}
function stopInputRecording () {
if ( inputRecorder ? . state === 'recording' ) inputRecorder . stop ();
}
function toggleInputRecording () {
if ( inputRecorder ? . state === 'recording' ) stopInputRecording ();
else startInputRecording ();
}
function updateRecordButton () {
const button = document . getElementById ( 'element2-copilot-record' );
if ( ! button ) return ;
const recording = inputRecorder ? . state === 'recording' ;
2026-07-13 12:12:04 +08:00
button . textContent = recording ? '■' : '●' ;
button . title = recording ? '停止录制 AI 输入' : '录制 AI 输入' ;
button . setAttribute ( 'aria-label' , button . title );
2026-07-09 08:41:57 +08:00
button . style . background = recording ? '#dc2626' : '#2563eb' ;
}
2026-07-13 12:12:04 +08:00
async function startSenderRecording () {
if ( senderRecorder ? . state === 'recording' ) return ;
selectBusinessSender ( 'record-sender' );
const track = selectedBusinessSender ? . track ;
if ( ! isLive ( track )) {
appendChatLine ( '录音' , '当前没有可录制的业务 sender 音轨' , true );
return ;
}
if ( ! window . MediaRecorder ) {
appendChatLine ( '录音' , '当前浏览器不支持 MediaRecorder' , true );
return ;
}
senderRecordingChunks = [];
senderRecordingTrack = track ;
senderRecordingStartedAt = Date . now ();
const mimeType = recorderMimeType ();
senderRecorder = new MediaRecorder ( new MediaStream ([ track ]), mimeType ? { mimeType } : undefined );
senderRecorder . ondataavailable = ( event ) => {
if ( event . data ? . size > 0 ) senderRecordingChunks . push ( event . data );
};
senderRecorder . onstop = () => {
const blob = new Blob ( senderRecordingChunks , { type : senderRecorder ? . mimeType || 'audio/webm' });
const url = URL . createObjectURL ( blob );
const link = document . createElement ( 'a' );
link . href = url ;
link . download = senderRecordingFileName ( senderRecordingTrack );
document . body . appendChild ( link );
link . click ();
link . remove ();
setTimeout (() => URL . revokeObjectURL ( url ), 30000 );
const seconds = (( Date . now () - senderRecordingStartedAt ) / 1000 ). toFixed ( 1 );
appendChatLine ( '录音' , `已保存 ${ link . download } ( ${ seconds } s, ${ ( blob . size / 1024 ). toFixed ( 1 ) } KB) ` , true );
senderRecorder = null ;
senderRecordingChunks = [];
senderRecordingTrack = null ;
updateSenderRecordButton ();
};
senderRecorder . start ( 250 );
appendChatLine ( '录音' , `开始录制业务 sender: ${ trackLabel ( track ) } ` , true );
updateSenderRecordButton ();
}
function toggleSenderRecording () {
if ( senderRecorder ? . state === 'recording' ) senderRecorder . stop ();
else void startSenderRecording ();
}
function updateSenderRecordButton () {
const button = document . getElementById ( 'element2-copilot-record-sender' );
if ( ! button ) return ;
const recording = senderRecorder ? . state === 'recording' ;
button . textContent = recording ? '■' : '●' ;
button . title = recording ? '停止录制 sender 音轨' : '录制 sender 音轨' ;
button . setAttribute ( 'aria-label' , button . title );
button . style . background = recording ? '#dc2626' : '#7c3aed' ;
}
2026-07-09 08:41:57 +08:00
function updateVideoRecordButton () {
const button = document . getElementById ( 'element2-copilot-record-video' );
if ( ! button ) return ;
const recording = videoRecorder ? . state === 'recording' ;
2026-07-13 12:12:04 +08:00
button . textContent = recording ? '■' : '●' ;
button . title = recording ? '停止录制 AI 视频' : '录制 AI 视频' ;
button . setAttribute ( 'aria-label' , button . title );
2026-07-09 08:41:57 +08:00
button . style . background = recording ? '#dc2626' : '#475569' ;
}
async function startVideoRecording () {
if ( videoRecorder ? . state === 'recording' ) return ;
selectRemoteVideo ( 'record-video' );
if ( ! isLive ( selectedRemoteVideo )) {
appendChatLine ( '录制视频' , `没有可录制的 ${ STATIC_VIDEO_ELEMENT_REASON } 视频轨` , true );
return ;
}
if ( ! window . MediaRecorder ) {
appendChatLine ( '录制视频' , '当前浏览器不支持 MediaRecorder' , true );
return ;
}
videoRecordingChunks = [];
videoRecordingTrack = selectedRemoteVideo ;
videoRecordingStartedAt = Date . now ();
const stream = new MediaStream ([ videoRecordingTrack ]);
const mimeType = videoRecorderMimeType ();
videoRecorder = new MediaRecorder ( stream , mimeType ? { mimeType } : undefined );
videoRecorder . ondataavailable = ( event ) => {
if ( event . data ? . size > 0 ) videoRecordingChunks . push ( event . data );
};
videoRecorder . onstop = () => {
const blob = new Blob ( videoRecordingChunks , { type : videoRecorder ? . mimeType || 'video/webm' });
const url = URL . createObjectURL ( blob );
const link = document . createElement ( 'a' );
link . href = url ;
link . download = videoRecordingFileName ( videoRecordingTrack );
document . body . appendChild ( link );
link . click ();
link . remove ();
setTimeout (() => URL . revokeObjectURL ( url ), 30000 );
const seconds = (( Date . now () - videoRecordingStartedAt ) / 1000 ). toFixed ( 1 );
appendChatLine ( '录制视频' , `已保存 ${ link . download } ( ${ seconds } s, ${ ( blob . size / 1024 ). toFixed ( 1 ) } KB) ` , true );
videoRecorder = null ;
videoRecordingChunks = [];
videoRecordingTrack = null ;
updateVideoRecordButton ();
};
videoRecorder . start ( 250 );
appendChatLine ( '录制视频' , `开始录制 AI 视频: ${ trackLabel ( videoRecordingTrack ) } ` , true );
updateVideoRecordButton ();
}
function stopVideoRecording () {
if ( videoRecorder ? . state === 'recording' ) videoRecorder . stop ();
}
function toggleVideoRecording () {
if ( videoRecorder ? . state === 'recording' ) stopVideoRecording ();
else startVideoRecording ();
}
function pcId () {
const bytes = new Uint8Array ( 16 );
crypto . getRandomValues ( bytes );
return 'PC-' + Array . from ( bytes ). map (( b ) => b . toString ( 16 ). padStart ( 2 , '0' )). join ( '' );
}
function isInternalPc ( pc ) {
return Boolean ( pc && ( pc . __aiElement2GatedInternal || internalPcs . has ( pc ) || pc === aiPc ));
}
function isLive ( track ) {
return track && track . readyState === 'live' ;
}
function trackLabel ( track ) {
if ( ! track ) return 'none' ;
return ` ${ track . id }${ track . muted ? ' muted' : '' } ` ;
}
function markInternalPc ( pc ) {
if ( ! pc ) return ;
pc . __aiElement2GatedInternal = true ;
internalPcs . add ( pc );
}
function elementReasonFor ( element ) {
if ( ! element ) return null ;
const tag = element . tagName ? . toLowerCase ();
if ( tag !== 'audio' && tag !== 'video' ) return null ;
const elements = Array . from ( document . querySelectorAll ( tag ));
const index = elements . indexOf ( element );
if ( index < 0 ) return ` ${ tag } -srcObject` ;
return `element- ${ index } : ${ tag } ` ;
}
function hasElementReason ( item , reason ) {
return Boolean ( item ? . reasons ? . has ( reason ));
}
function elementAudioReasons ( item ) {
return Array . from ( item ? . reasons || []). filter (( reason ) => /^ element - \ d +: audio$ / . test ( reason ));
}
function audioSourceLabel ( item ) {
const reasons = elementAudioReasons ( item );
return reasons [ 0 ] || 'audio-track' ;
}
2026-07-13 12:12:04 +08:00
function shortTrackId ( track ) {
if ( ! track ? . id ) return 'none' ;
return track . id . length <= 22 ? track . id : ` ${ track . id . slice ( 0 , 10 ) } ... ${ track . id . slice ( - 7 ) } ` ;
}
function selectableAudioItems () {
return Array . from ( remoteAudio . values ()). filter (( item ) => (
item ? . receiver
&& isLive ( item . track )
&& ! aiOutputTrackIds . has ( item . track . id )
));
}
function audioOptionLabel ( item ) {
const source = audioSourceLabel ( item );
const state = item . track . muted ? '静音' : '可用' ;
return ` ${ source } · ${ shortTrackId ( item . track ) } · ${ state } ` ;
}
function updateAudioTrackSelect () {
const select = document . getElementById ( 'element2-copilot-audio-track' );
if ( ! select ) return ;
const items = selectableAudioItems (). sort (( a , b ) => scoreAudioCandidate ( b ) - scoreAudioCandidate ( a ));
const desiredValue = audioSelectionMode === 'manual' ? manualAudioTrackId : selectedRemoteAudio ? . id ;
const signature = [
audioSelectionMode ,
desiredValue || '' ,
... items . map (( item ) => ` ${ item . track . id } : ${ item . track . muted } : ${ audioSourceLabel ( item ) } ` ),
]. join ( '|' );
if ( signature === audioTrackSelectSignature ) return ;
audioTrackSelectSignature = signature ;
select . innerHTML = '' ;
if ( items . length === 0 ) {
const option = document . createElement ( 'option' );
option . value = '' ;
option . textContent = '暂无可用远端音轨' ;
select . appendChild ( option );
} else {
if ( audioSelectionMode === 'manual' && ! manualAudioTrackId ) {
const placeholder = document . createElement ( 'option' );
placeholder . value = '' ;
placeholder . textContent = '请选择远端音轨' ;
select . appendChild ( placeholder );
}
for ( const item of items ) {
const option = document . createElement ( 'option' );
option . value = item . track . id ;
option . textContent = audioOptionLabel ( item );
select . appendChild ( option );
}
}
if ( desiredValue && items . some (( item ) => item . track . id === desiredValue )) select . value = desiredValue ;
select . disabled = audioSelectionMode !== 'manual' ;
select . style . opacity = select . disabled ? '0.55' : '1' ;
}
2026-07-09 08:41:57 +08:00
function updateSelectedAudioLabel () {
const label = document . getElementById ( 'element2-copilot-selected-audio' );
if ( ! label ) return ;
2026-07-13 12:12:04 +08:00
const mode = audioSelectionMode === 'manual' ? '手动' : audioSelectionMode === 'off' ? '关闭' : '自动' ;
const item = selectedRemoteAudio ? remoteAudio . get ( selectedRemoteAudio . id ) : null ;
label . textContent = selectedRemoteAudio
? ` ${ mode } : ${ audioSourceLabel ( item ) } · ${ shortTrackId ( selectedRemoteAudio ) } `
: audioSelectionMode === 'off' ? '不传音频给 AI' : ` ${ mode } :尚未选中音轨` ;
2026-07-09 08:41:57 +08:00
}
2026-07-09 08:54:15 +08:00
function pageHasParticipant () {
2026-07-09 08:41:57 +08:00
const text = document . body ? . innerText || '' ;
2026-07-09 08:54:15 +08:00
return text . includes ( '当事人' );
}
function pageHasOperator () {
const text = document . body ? . innerText || '' ;
return text . includes ( '坐席端' );
}
2026-07-13 12:12:04 +08:00
function selectedAudioItem () {
const item = selectedRemoteAudio ? remoteAudio . get ( selectedRemoteAudio . id ) : null ;
return item ? . receiver && isLive ( item . track ) && ! item . track . muted ? item : null ;
2026-07-09 08:41:57 +08:00
}
function gateStatus () {
2026-07-13 12:12:04 +08:00
const item = selectedAudioItem ();
const participantAudioPresent = selectableAudioItems (). some (( candidate ) => ! candidate . track . muted );
const senderItem = Array . from ( businessSenders . values ()). find (( candidate ) => (
candidate . sender === selectedBusinessSender
&& selectableSenderItems (). includes ( candidate )
));
2026-07-09 08:54:15 +08:00
const participantVisible = pageHasParticipant ();
const operatorVisible = pageHasOperator ();
2026-07-09 08:41:57 +08:00
return {
2026-07-09 08:54:15 +08:00
bothParties : participantVisible && operatorVisible ,
2026-07-13 12:12:04 +08:00
participantOnline : participantVisible && ( Boolean ( item ) || ( audioSelectionMode === 'off' && participantAudioPresent )),
2026-07-09 08:54:15 +08:00
operatorOnline : operatorVisible ,
2026-07-13 12:12:04 +08:00
senderReady : Boolean ( senderItem ),
2026-07-09 08:41:57 +08:00
track : item ? . track || null ,
};
}
function gateMessage ( status = gateStatus ()) {
2026-07-09 08:54:15 +08:00
if ( ! status . operatorOnline ) return '等待坐席在线' ;
if ( ! status . participantOnline ) return '等待当事人在线' ;
if ( ! status . senderReady ) return '等待通话上行就绪' ;
return '就绪:可开启 AI 托管' ;
2026-07-09 08:41:57 +08:00
}
function updateGateLabel () {
const label = document . getElementById ( 'element2-copilot-gate' );
if ( ! label ) return ;
const status = gateStatus ();
label . textContent = gateMessage ( status );
2026-07-09 08:54:15 +08:00
label . style . color = status . participantOnline && status . operatorOnline && status . senderReady ? '#86efac' : '#fcd34d' ;
2026-07-09 08:41:57 +08:00
}
function scoreAudioCandidate ( item ) {
if ( ! item || ! isLive ( item . track ) || item . track . muted || aiOutputTrackIds . has ( item . track . id )) return - 1 ;
let score = 0 ;
if ( item . receiver ) score += 300 ;
if (( item . bytesReceived || 0 ) > 0 ) score += 100 ;
if (( item . bytesDelta || 0 ) > 0 ) score += 4000 + Math . min ( 1500 , item . bytesDelta );
if (( item . packetsDelta || 0 ) > 0 ) score += 2000 + Math . min ( 800 , item . packetsDelta * 8 );
score += Math . min ( 1200 , ( item . audioLevel || 0 ) * 12000 );
if ( elementAudioReasons ( item ). length > 0 ) score += 100 ;
2026-07-13 12:12:04 +08:00
if ( hasElementReason ( item , STATIC_AUDIO_ELEMENT_REASON )) score += 5000 ;
if ( PARTICIPANT_AUDIO_TRACK_PATTERN . test ( item . track . id )) score += 5000 ;
2026-07-09 08:41:57 +08:00
item . audioScore = score ;
return score ;
}
function rememberRemoteReceiver ( receiver , reason ) {
const track = receiver ? . track ;
if ( ! isLive ( track ) || isInternalPc ( receiver ? . transport )) return ;
if ( track . kind === 'audio' && ! aiOutputTrackIds . has ( track . id )) {
let item = remoteAudio . get ( track . id );
if ( ! item ) {
item = { track , receiver , reasons : new Set (), bytesDelta : 0 , packetsDelta : 0 , audioLevel : 0 , audioScore : 0 };
remoteAudio . set ( track . id , item );
log ( `audio candidate ${ trackLabel ( track ) } ( ${ reason } )` , true );
}
item . track = track ;
item . receiver = receiver || item . receiver ;
item . reasons . add ( reason );
selectRemoteAudio ( reason );
}
if ( track . kind === 'video' ) {
let item = remoteVideo . get ( track . id );
if ( ! item ) {
item = { track , receiver , reasons : new Set () };
remoteVideo . set ( track . id , item );
log ( `video candidate ${ trackLabel ( track ) } ( ${ reason } )` , true );
}
item . track = track ;
item . receiver = receiver || item . receiver ;
item . reasons . add ( reason );
selectRemoteVideo ( reason );
}
}
function rememberRemoteStream ( stream , reason ) {
if ( ! stream || ! ( stream instanceof MediaStream )) return ;
for ( const track of stream . getAudioTracks ()) {
if ( aiOutputTrackIds . has ( track . id )) continue ;
let item = remoteAudio . get ( track . id );
if ( ! item ) {
item = { track , receiver : null , reasons : new Set (), bytesDelta : 0 , packetsDelta : 0 , audioLevel : 0 , audioScore : 0 };
remoteAudio . set ( track . id , item );
log ( `audio stream candidate ${ trackLabel ( track ) } ( ${ reason } )` , true );
}
item . track = track ;
item . reasons . add ( reason );
}
for ( const track of stream . getVideoTracks ()) {
let item = remoteVideo . get ( track . id );
if ( ! item ) {
item = { track , receiver : null , reasons : new Set () };
remoteVideo . set ( track . id , item );
log ( `video stream candidate ${ trackLabel ( track ) } ( ${ reason } )` , true );
}
item . track = track ;
item . reasons . add ( reason );
}
selectRemoteAudio ( reason );
selectRemoteVideo ( reason );
}
function selectRemoteAudio ( reason ) {
2026-07-13 12:12:04 +08:00
const items = selectableAudioItems ();
2026-07-09 08:41:57 +08:00
const current = selectedRemoteAudio && remoteAudio . get ( selectedRemoteAudio . id );
2026-07-13 12:12:04 +08:00
let item = null ;
if ( audioSelectionMode === 'off' ) {
item = null ;
} else if ( audioSelectionMode === 'manual' ) {
item = items . find (( candidate ) => candidate . track . id === manualAudioTrackId ) || null ;
} else {
const ranked = items
. filter (( candidate ) => ! candidate . track . muted )
. sort (( a , b ) => scoreAudioCandidate ( b ) - scoreAudioCandidate ( a ));
const best = ranked [ 0 ] || null ;
const currentScore = current ? scoreAudioCandidate ( current ) : - 1 ;
const bestScore = best ? scoreAudioCandidate ( best ) : - 1 ;
item = current && currentScore >= 0 && ( ! best || bestScore < currentScore + 1500 )
? current
: best ;
2026-07-09 08:41:57 +08:00
}
if ( ! item ) {
2026-07-13 12:12:04 +08:00
if ( selectedRemoteAudio ) {
selectedRemoteAudio = null ;
if ( aiAudioSender ) void aiAudioSender . replaceTrack ( null ). catch (() => {});
}
2026-07-09 08:41:57 +08:00
updateSelectedAudioLabel ();
2026-07-13 12:12:04 +08:00
updateAudioTrackSelect ();
2026-07-09 08:41:57 +08:00
updateGateLabel ();
return null ;
}
if ( selectedRemoteAudio !== item . track ) {
selectedRemoteAudio = item . track ;
2026-07-13 12:12:04 +08:00
log ( `selected input audio ( ${ reason } ) from ${ audioSourceLabel ( item ) } : ${ trackLabel ( selectedRemoteAudio ) } ` , true );
2026-07-09 08:41:57 +08:00
updateSelectedAudioLabel ();
attachAudioToAi ();
}
2026-07-13 12:12:04 +08:00
updateAudioTrackSelect ();
2026-07-09 08:41:57 +08:00
updateGateLabel ();
return selectedRemoteAudio ;
}
function selectRemoteVideo ( reason ) {
2026-07-13 12:12:04 +08:00
const items = selectableVideoItems ();
let item = null ;
if ( videoSelectionMode === 'manual' ) {
item = items . find (( candidate ) => candidate . track . id === manualVideoTrackId ) || null ;
} else if ( videoSelectionMode === 'auto' ) {
item = items . find (( candidate ) => hasElementReason ( candidate , STATIC_VIDEO_ELEMENT_REASON )) || items [ 0 ] || null ;
}
if ( ! item ) {
if ( selectedRemoteVideo ) {
selectedRemoteVideo = null ;
if ( aiVideoSender ) void aiVideoSender . replaceTrack ( null ). catch (() => {});
}
updateVideoTrackSelect ();
return null ;
2026-07-09 08:41:57 +08:00
}
if ( selectedRemoteVideo !== item . track ) {
selectedRemoteVideo = item . track ;
2026-07-13 12:12:04 +08:00
log ( `selected input video ( ${ reason } ): ${ trackLabel ( selectedRemoteVideo ) } ` , true );
2026-07-09 08:41:57 +08:00
attachVideoToAi ();
}
2026-07-13 12:12:04 +08:00
updateVideoTrackSelect ();
2026-07-09 08:41:57 +08:00
return selectedRemoteVideo ;
}
2026-07-13 12:12:04 +08:00
function selectableVideoItems () {
return Array . from ( remoteVideo . values ()). filter (( item ) => (
item ? . receiver
&& isLive ( item . track )
&& ! item . track . muted
));
}
function videoSourceLabel ( item ) {
const reason = Array . from ( item ? . reasons || []). find (( value ) => /^ element - \ d +: video$ / . test ( value ));
return reason || 'video-track' ;
}
function updateVideoTrackSelect () {
const select = document . getElementById ( 'element2-copilot-video-track' );
if ( ! select ) return ;
const items = selectableVideoItems ();
const desiredValue = videoSelectionMode === 'manual' ? manualVideoTrackId : selectedRemoteVideo ? . id ;
const signature = [
videoSelectionMode ,
desiredValue || '' ,
... items . map (( item ) => ` ${ item . track . id } : ${ videoSourceLabel ( item ) } ` ),
]. join ( '|' );
if ( signature === videoTrackSelectSignature ) return ;
videoTrackSelectSignature = signature ;
select . innerHTML = '' ;
if ( items . length === 0 ) {
const option = document . createElement ( 'option' );
option . value = '' ;
option . textContent = '暂无可用远端视频' ;
select . appendChild ( option );
} else {
if ( videoSelectionMode === 'manual' && ! manualVideoTrackId ) {
const placeholder = document . createElement ( 'option' );
placeholder . value = '' ;
placeholder . textContent = '请选择远端视频' ;
select . appendChild ( placeholder );
}
for ( const item of items ) {
const option = document . createElement ( 'option' );
option . value = item . track . id ;
option . textContent = ` ${ videoSourceLabel ( item ) } · ${ shortTrackId ( item . track ) } ` ;
select . appendChild ( option );
}
}
if ( desiredValue && items . some (( item ) => item . track . id === desiredValue )) select . value = desiredValue ;
select . disabled = videoSelectionMode !== 'manual' ;
select . style . opacity = select . disabled ? '0.55' : '1' ;
}
2026-07-09 08:41:57 +08:00
function rememberBusinessSender ( sender , track , reason ) {
if ( ! sender ) return ;
const senderTrack = track || sender . track ;
if ( senderTrack && senderTrack . kind !== 'audio' ) return ;
const id = sender . __aiElement2GatedSenderId || `sender- ${ businessSenders . size + 1 } ` ;
sender . __aiElement2GatedSenderId = id ;
let item = businessSenders . get ( id );
if ( ! item ) {
item = { id , sender , track : senderTrack || null , originalTrack : senderTrack || null , sources : new Set (), bytesDelta : 0 , packetsDelta : 0 };
businessSenders . set ( id , item );
log ( `business sender candidate ${ id } ( ${ reason } ): ${ trackLabel ( senderTrack ) } ` , true );
}
item . sender = sender ;
item . track = sender . track || senderTrack || item . track ;
if ( ! isAiTakeover && senderTrack && senderTrack !== aiAudioTrack ) item . originalTrack = senderTrack ;
item . sources . add ( reason );
wrapBusinessSender ( sender );
selectBusinessSender ( reason );
}
function wrapBusinessSender ( sender ) {
if ( ! sender || sender . __aiElement2GatedWrapped ) return ;
const originalReplaceTrack = sender . replaceTrack ? . bind ( sender );
if ( ! originalReplaceTrack ) return ;
sender . replaceTrack = async function ( track ) {
const result = await originalReplaceTrack ( track );
rememberBusinessSender ( sender , track , 'replaceTrack' );
return result ;
};
sender . __aiElement2GatedWrapped = true ;
}
function isRemoteInputTrack ( track ) {
2026-07-13 12:12:04 +08:00
const item = track ? remoteAudio . get ( track . id ) : null ;
return Boolean ( item ? . receiver && ! aiOutputTrackIds . has ( track . id ));
}
function senderCandidateTrack ( item ) {
const current = item ? . sender ? . track ;
if ( current && current !== aiAudioTrack ) return current ;
return item ? . originalTrack || item ? . track || null ;
}
function selectableSenderItems () {
return Array . from ( businessSenders . values ()). filter (( item ) => {
const track = senderCandidateTrack ( item );
return isLive ( track ) && track . kind === 'audio' && ! isRemoteInputTrack ( track );
});
}
function scoreSenderCandidate ( item ) {
const track = senderCandidateTrack ( item );
if ( ! isLive ( track ) || isRemoteInputTrack ( track )) return - 1 ;
let score = 1000 ;
if ( ! track . muted ) score += 400 ;
if ( item . sources . has ( 'pc-addTrack' )) score += 2000 ;
if (( item . bytesSent || 0 ) > 0 ) score += 300 ;
if (( item . bytesDelta || 0 ) > 0 ) score += 4500 + Math . min ( 1800 , item . bytesDelta );
if (( item . packetsDelta || 0 ) > 0 ) score += 1800 + Math . min ( 900 , item . packetsDelta * 10 );
return score ;
}
function updateSenderSelect () {
const select = document . getElementById ( 'element2-copilot-sender' );
if ( ! select ) return ;
const items = selectableSenderItems (). sort (( a , b ) => scoreSenderCandidate ( b ) - scoreSenderCandidate ( a ));
const selectedItem = items . find (( item ) => item . sender === selectedBusinessSender );
const desiredValue = senderSelectionMode === 'manual' ? manualSenderId : selectedItem ? . id ;
const signature = [
senderSelectionMode ,
desiredValue || '' ,
... items . map (( item ) => ` ${ item . id } : ${ senderCandidateTrack ( item ) ? . id || '' } : ${ senderCandidateTrack ( item ) ? . muted } ` ),
]. join ( '|' );
if ( signature === senderSelectSignature ) return ;
senderSelectSignature = signature ;
select . innerHTML = '' ;
if ( items . length === 0 ) {
const option = document . createElement ( 'option' );
option . value = '' ;
option . textContent = '暂无可用 sender' ;
select . appendChild ( option );
} else {
if ( senderSelectionMode === 'manual' && ! manualSenderId ) {
const placeholder = document . createElement ( 'option' );
placeholder . value = '' ;
placeholder . textContent = '请选择 sender' ;
select . appendChild ( placeholder );
}
for ( const item of items ) {
const option = document . createElement ( 'option' );
option . value = item . id ;
option . textContent = ` ${ item . id } · ${ shortTrackId ( senderCandidateTrack ( item )) }${ item . sources . has ( 'pc-addTrack' ) ? ' · addTrack' : '' } ` ;
select . appendChild ( option );
}
}
if ( desiredValue && items . some (( item ) => item . id === desiredValue )) select . value = desiredValue ;
select . disabled = senderSelectionMode !== 'manual' ;
select . style . opacity = select . disabled ? '0.55' : '1' ;
2026-07-09 08:41:57 +08:00
}
function selectBusinessSender ( reason ) {
2026-07-13 12:12:04 +08:00
if ( changingBusinessSender ) return selectedBusinessSender ;
const items = selectableSenderItems ();
const current = items . find (( candidate ) => candidate . sender === selectedBusinessSender );
let item = null ;
if ( senderSelectionMode === 'manual' ) {
item = items . find (( candidate ) => candidate . id === manualSenderId ) || null ;
} else {
item = current || items . sort (( a , b ) => scoreSenderCandidate ( b ) - scoreSenderCandidate ( a ))[ 0 ] || null ;
}
if ( ! item ) {
if ( ! aiAttached ) {
selectedBusinessSender = null ;
originalBusinessTrack = null ;
}
updateSenderSelect ();
return null ;
}
if ( selectedBusinessSender === item . sender ) {
updateSenderSelect ();
return selectedBusinessSender ;
}
2026-07-09 08:41:57 +08:00
selectedBusinessSender = item . sender ;
originalBusinessTrack = item . originalTrack || item . sender . track || originalBusinessTrack ;
log ( `selected output sender ( ${ reason } ): ${ item . id } , track= ${ trackLabel ( item . sender . track ) } ` , true );
2026-07-13 12:12:04 +08:00
updateSenderSelect ();
2026-07-09 08:41:57 +08:00
updateGateLabel ();
attachAiOutputToBusiness ();
return selectedBusinessSender ;
}
2026-07-13 12:12:04 +08:00
async function changeBusinessSender ( reason ) {
changingBusinessSender = true ;
try {
await restoreBusinessAudio ();
selectedBusinessSender = null ;
originalBusinessTrack = null ;
aiAttached = false ;
} finally {
changingBusinessSender = false ;
}
senderSelectSignature = '' ;
selectBusinessSender ( reason );
if ( isAiTakeover ) await attachAiOutputToBusiness ();
updateGateLabel ();
}
2026-07-09 08:41:57 +08:00
function setVu ( id , level ) {
const el = document . getElementById ( id );
2026-07-09 08:54:15 +08:00
if ( ! el ) return ;
el . style . width = ` ${ Math . min ( 100 , level * 2500 ) } %` ;
el . style . opacity = level > 0 ? '1' : '0.35' ;
}
function volumePercent ( level ) {
if ( ! Number . isFinite ( level ) || level < 0.0015 ) return 0 ;
return Math . round ( Math . min ( 100 , level * 2500 ));
}
function volumeDb ( level ) {
if ( ! Number . isFinite ( level ) || level < 0.0015 ) return '-inf' ;
return ` ${ ( 20 * Math . log10 ( level )). toFixed ( 1 ) } dB` ;
}
function volumeText ( level ) {
return `vol= ${ volumePercent ( level ) } % ${ volumeDb ( level ) } ` ;
}
function setVolumeLabel ( id , level ) {
const el = document . getElementById ( id );
if ( ! el ) return ;
el . textContent = volumeText ( level );
2026-07-09 08:41:57 +08:00
}
function renderVu () {
2026-07-09 08:54:15 +08:00
refreshAiOutputVuFromAnalyser ();
2026-07-09 08:41:57 +08:00
inputVu *= 0.88 ;
outputVu *= 0.88 ;
setVu ( 'element2-copilot-input-vu' , inputVu );
setVu ( 'element2-copilot-output-vu' , outputVu );
2026-07-09 08:54:15 +08:00
setVolumeLabel ( 'element2-copilot-input-volume' , inputVu );
setVolumeLabel ( 'element2-copilot-output-volume' , outputVu );
2026-07-09 08:41:57 +08:00
requestAnimationFrame ( renderVu );
}
2026-07-09 08:54:15 +08:00
function speechVuLevel ( level ) {
const noiseFloor = 0.0015 ;
if ( ! Number . isFinite ( level ) || level < noiseFloor ) return 0 ;
return Math . min ( 0.08 , ( level - noiseFloor ) * 3.2 );
}
function analyserVuLevel ( analyser , data ) {
if ( ! analyser || ! data ) return 0 ;
analyser . getByteTimeDomainData ( data );
let sum = 0 ;
for ( const value of data ) {
const centered = ( value - 128 ) / 128 ;
sum += centered * centered ;
}
return speechVuLevel ( Math . sqrt ( sum / data . length ));
}
function refreshAiOutputVuFromAnalyser () {
outputVu = Math . max ( outputVu , analyserVuLevel ( aiOutputAnalyser , aiOutputAnalyserData ));
}
function createAudioContext () {
const AudioContextCtor = window . AudioContext || window . webkitAudioContext ;
return AudioContextCtor ? new AudioContextCtor () : null ;
}
function detachAiOutputMeter () {
try {
aiOutputAnalyserSource ? . disconnect ();
} catch {}
aiOutputAnalyser = null ;
aiOutputAnalyserData = null ;
aiOutputAnalyserSource = null ;
aiOutputAnalyserTrack = null ;
outputVu = 0 ;
}
async function attachAiOutputMeter ( track ) {
if ( ! isLive ( track )) {
detachAiOutputMeter ();
return false ;
}
if ( aiOutputAnalyserTrack === track && aiOutputAnalyser ) return true ;
detachAiOutputMeter ();
if ( ! aiOutputAudioContext ) {
aiOutputAudioContext = createAudioContext ();
if ( ! aiOutputAudioContext ) {
appendChatLine ( '音频' , '当前浏览器不支持 AudioContext,无法显示 AI 输出音量' , true );
return false ;
}
}
if ( aiOutputAudioContext . state === 'suspended' ) {
await aiOutputAudioContext . resume (). catch (() => {});
}
try {
const stream = new MediaStream ([ track ]);
aiOutputAnalyserSource = aiOutputAudioContext . createMediaStreamSource ( stream );
aiOutputAnalyser = aiOutputAudioContext . createAnalyser ();
aiOutputAnalyser . fftSize = 512 ;
aiOutputAnalyser . smoothingTimeConstant = 0.55 ;
aiOutputAnalyserData = new Uint8Array ( aiOutputAnalyser . fftSize );
aiOutputAnalyserSource . connect ( aiOutputAnalyser );
aiOutputAnalyserTrack = track ;
return true ;
} catch ( error ) {
log ( `failed to attach AI output meter: ${ error . message || error } ` );
detachAiOutputMeter ();
return false ;
}
}
2026-07-09 08:41:57 +08:00
async function refreshStats () {
await refreshInputStats ();
await refreshOutputStats ();
}
async function refreshInputStats () {
for ( const item of remoteAudio . values ()) {
if ( ! item . receiver ? . getStats ) continue ;
const report = await item . receiver . getStats ();
report . forEach (( stat ) => {
const kind = stat . kind || stat . mediaType ;
if ( stat . type !== 'inbound-rtp' || kind !== 'audio' ) return ;
const prev = inboundMemory . get ( stat . id );
const bytes = stat . bytesReceived || 0 ;
const packets = stat . packetsReceived || 0 ;
let audioLevel = typeof stat . audioLevel === 'number' ? stat . audioLevel : 0 ;
if (
prev
&& typeof stat . totalAudioEnergy === 'number'
&& typeof stat . totalSamplesDuration === 'number'
&& typeof prev . energy === 'number'
&& typeof prev . duration === 'number'
) {
const energyDelta = stat . totalAudioEnergy - prev . energy ;
const durationDelta = stat . totalSamplesDuration - prev . duration ;
if ( energyDelta > 0 && durationDelta > 0 ) {
audioLevel = Math . max ( audioLevel , Math . sqrt ( energyDelta / durationDelta ));
}
}
item . bytesReceived = bytes ;
item . packetsReceived = packets ;
item . bytesDelta = prev ? Math . max ( 0 , bytes - prev . bytes ) : 0 ;
item . packetsDelta = prev ? Math . max ( 0 , packets - prev . packets ) : 0 ;
item . audioLevel = audioLevel ;
inboundMemory . set ( stat . id , {
bytes ,
packets ,
energy : stat . totalAudioEnergy ,
duration : stat . totalSamplesDuration ,
});
if ( item . track === selectedRemoteAudio ) {
2026-07-09 08:54:15 +08:00
inputVu = Math . max ( inputVu , speechVuLevel ( audioLevel ));
2026-07-09 08:41:57 +08:00
}
});
}
selectRemoteAudio ( 'stats' );
}
async function refreshOutputStats () {
for ( const item of businessSenders . values ()) {
if ( ! item . sender ? . getStats ) continue ;
item . track = item . sender . track || item . track ;
const report = await item . sender . getStats ();
report . forEach (( stat ) => {
const kind = stat . kind || stat . mediaType ;
if ( stat . type !== 'outbound-rtp' || kind !== 'audio' ) return ;
const key = ` ${ item . id } : ${ stat . id } ` ;
const prev = outboundMemory . get ( key );
const bytes = stat . bytesSent || 0 ;
const packets = stat . packetsSent || 0 ;
item . bytesSent = bytes ;
item . packetsSent = packets ;
item . bytesDelta = prev ? Math . max ( 0 , bytes - prev . bytes ) : 0 ;
item . packetsDelta = prev ? Math . max ( 0 , packets - prev . packets ) : 0 ;
outboundMemory . set ( key , { bytes , packets });
});
}
2026-07-13 12:12:04 +08:00
selectBusinessSender ( 'stats' );
updateSenderSelect ();
2026-07-09 08:41:57 +08:00
}
async function outboundBytes ( sender ) {
if ( ! sender ? . getStats ) return null ;
let total = 0 ;
const report = await sender . getStats ();
report . forEach (( stat ) => {
const kind = stat . kind || stat . mediaType ;
if ( stat . type === 'outbound-rtp' && kind === 'audio' ) total += stat . bytesSent || 0 ;
});
return total ;
}
function aiState () {
if ( ! aiPc ) return 'pc=null' ;
return `conn= ${ aiPc . connectionState } ,ice= ${ aiPc . iceConnectionState } ,signaling= ${ aiPc . signalingState } ,remote= ${ aiPc . remoteDescription ? . type || 'none' } ` ;
}
function aiReady () {
return Boolean ( aiPc ? . remoteDescription && (
aiPc . connectionState === 'connected'
|| aiPc . iceConnectionState === 'connected'
|| aiPc . iceConnectionState === 'completed'
));
}
function waitForAiReady ( timeoutMs = 10000 ) {
if ( aiReady ()) return Promise . resolve ( true );
return new Promise (( resolve ) => {
const startedAt = Date . now ();
const timer = setInterval (() => {
if ( aiReady ()) {
clearInterval ( timer );
resolve ( true );
} else if ( Date . now () - startedAt >= timeoutMs ) {
clearInterval ( timer );
resolve ( false );
}
}, 250 );
});
}
function checkAiInput ( track ) {
if ( inputHealthTimer ) clearTimeout ( inputHealthTimer );
inputHealthTimer = setTimeout ( async () => {
if ( ! aiAudioSender || aiAudioSender . track !== track ) return ;
await waitForAiReady ();
const before = await outboundBytes ( aiAudioSender );
await new Promise (( resolve ) => setTimeout ( resolve , 2500 ));
const after = await outboundBytes ( aiAudioSender );
log ( `AI input packet check: before= ${ before } , after= ${ after } , ${ aiState () } ` , true );
}, 500 );
}
function checkAiOutput ( track ) {
if ( outputHealthTimer ) clearTimeout ( outputHealthTimer );
outputHealthTimer = setTimeout ( async () => {
if ( ! selectedBusinessSender || selectedBusinessSender . track !== track ) return ;
const before = await outboundBytes ( selectedBusinessSender );
await new Promise (( resolve ) => setTimeout ( resolve , 2500 ));
const after = await outboundBytes ( selectedBusinessSender );
log ( `AI output packet check: before= ${ before } , after= ${ after } , sender= ${ trackLabel ( selectedBusinessSender . track ) } ` , true );
}, 800 );
}
async function attachAudioToAi () {
if ( ! shouldSendAudioToAi ()) return false ;
if ( ! aiAudioSender || ! isLive ( selectedRemoteAudio )) return false ;
if ( aiAudioSender . track === selectedRemoteAudio ) return true ;
await aiAudioSender . replaceTrack ( selectedRemoteAudio );
log ( `AI input uses ${ trackLabel ( selectedRemoteAudio ) } from ${ STATIC_AUDIO_ELEMENT_REASON } ` , true );
checkAiInput ( selectedRemoteAudio );
return true ;
}
async function attachVideoToAi () {
if ( ! shouldSendVideoToAi ()) return false ;
if ( ! aiVideoSender || ! isLive ( selectedRemoteVideo )) return false ;
if ( aiVideoSender . track === selectedRemoteVideo ) return true ;
await aiVideoSender . replaceTrack ( selectedRemoteVideo );
log ( `AI video uses ${ trackLabel ( selectedRemoteVideo ) } from ${ STATIC_VIDEO_ELEMENT_REASON } ` , true );
return true ;
}
async function attachAiOutputToBusiness () {
selectBusinessSender ( 'before-output' );
if ( ! isAiTakeover || ! selectedBusinessSender || ! isLive ( aiAudioTrack )) {
log ( `AI output waiting: takeover= ${ isAiTakeover } , sender= ${ Boolean ( selectedBusinessSender ) } , aiTrack= ${ trackLabel ( aiAudioTrack ) } ` );
return false ;
}
if ( selectedBusinessSender . track === aiAudioTrack ) return true ;
await selectedBusinessSender . replaceTrack ( aiAudioTrack );
aiAttached = true ;
log ( `business sender uses AI audio ${ trackLabel ( aiAudioTrack ) } ` , true );
checkAiOutput ( aiAudioTrack );
return true ;
}
async function restoreBusinessAudio () {
if ( ! selectedBusinessSender || ! aiAttached ) return ;
await selectedBusinessSender . replaceTrack ( originalBusinessTrack || null );
aiAttached = false ;
log ( `business sender restored ${ trackLabel ( originalBusinessTrack ) } ` , true );
}
function wrapTrackListener ( listener ) {
if ( ! listener || listener . __aiElement2GatedWrapped ) return listener ;
const wrapped = function ( event ) {
const ownerPc = this instanceof RTCPeerConnection ? this : event ? . currentTarget ;
if ( ! isInternalPc ( ownerPc ) && event ? . track ) {
rememberRemoteReceiver ( event . receiver , 'pc-track' );
for ( const stream of event . streams || []) rememberRemoteStream ( stream , 'pc-track-stream' );
}
if ( typeof listener === 'function' ) return listener . apply ( this , arguments );
if ( listener ? . handleEvent ) return listener . handleEvent ( event );
return undefined ;
};
wrapped . __aiElement2GatedWrapped = true ;
return wrapped ;
}
RTCPeerConnection . prototype . addEventListener = function ( type , listener , options ) {
if ( type === 'track' ) return originalAddEventListener . call ( this , type , wrapTrackListener ( listener ), options );
return originalAddEventListener . call ( this , type , listener , options );
};
try {
const descriptor = Object . getOwnPropertyDescriptor ( RTCPeerConnection . prototype , 'ontrack' );
Object . defineProperty ( RTCPeerConnection . prototype , 'ontrack' , {
configurable : true ,
enumerable : true ,
get () {
return descriptor ? . get ? descriptor . get . call ( this ) : this . __aiElement2GatedOnTrack || null ;
},
set ( listener ) {
const wrapped = wrapTrackListener ( listener );
this . __aiElement2GatedOnTrack = wrapped ;
if ( descriptor ? . set ) descriptor . set . call ( this , wrapped );
else this . addEventListener ( 'track' , wrapped );
},
});
} catch ( error ) {
console . warn ( '[AI Element2 Gated] failed to wrap ontrack' , error );
}
RTCPeerConnection . prototype . addTrack = function ( track , ... streams ) {
const sender = originalAddTrack . call ( this , track , ... streams );
if ( ! isInternalPc ( this ) && track ? . kind === 'audio' ) rememberBusinessSender ( sender , track , 'pc-addTrack' );
return sender ;
};
if ( originalAddTransceiver ) {
RTCPeerConnection . prototype . addTransceiver = function ( trackOrKind , init ) {
const transceiver = originalAddTransceiver . call ( this , trackOrKind , init );
if ( ! isInternalPc ( this )) {
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind . kind : trackOrKind ;
if ( kind === 'audio' ) {
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver . sender ? . track ;
rememberBusinessSender ( transceiver . sender , track , 'pc-addTransceiver' );
}
}
return transceiver ;
};
}
RTCPeerConnection . prototype . setRemoteDescription = function () {
const result = originalSetRemoteDescription . apply ( this , arguments );
if ( ! isInternalPc ( this )) {
Promise . resolve ( result ). then (() => {
for ( const receiver of this . getReceivers ? .() || []) rememberRemoteReceiver ( receiver , 'receiver-after-srd' );
for ( const sender of this . getSenders ? .() || []) rememberBusinessSender ( sender , sender . track , 'sender-after-srd' );
}). catch (() => {});
}
return result ;
};
if ( mediaSrcObjectDescriptor ? . set ) {
Object . defineProperty ( HTMLMediaElement . prototype , 'srcObject' , {
configurable : true ,
enumerable : mediaSrcObjectDescriptor . enumerable ,
get () {
return mediaSrcObjectDescriptor . get . call ( this );
},
set ( stream ) {
if ( ! this . __aiElement2GatedCopilotElement ) {
const reason = elementReasonFor ( this ) || ` ${ this . tagName . toLowerCase () } -srcObject` ;
rememberRemoteStream ( stream , reason );
}
return mediaSrcObjectDescriptor . set . call ( this , stream );
},
});
}
function scanMediaElements () {
document . querySelectorAll ( 'audio' ). forEach (( element , index ) => {
if ( element . __aiElement2GatedCopilotElement ) return ;
rememberRemoteStream ( element . srcObject , `element- ${ index } :audio` );
});
document . querySelectorAll ( 'video' ). forEach (( element , index ) => {
if ( element . __aiElement2GatedCopilotElement ) return ;
rememberRemoteStream ( element . srcObject , `element- ${ index } :video` );
});
selectRemoteAudio ( 'media-scan' );
selectRemoteVideo ( 'media-scan' );
updateGateLabel ();
}
setInterval ( scanMediaElements , 1000 );
2026-07-13 12:12:04 +08:00
function endpointUrl ( baseUrl , path ) {
const normalizedBaseUrl = baseUrl . trim (). replace ( /\/+$/ , '' );
return new URL ( ` ${ normalizedBaseUrl }${ path } ` ). toString ();
}
function basicAuthorization ( username , password ) {
const bytes = new TextEncoder (). encode ( ` ${ username } : ${ password } ` );
let binary = '' ;
bytes . forEach (( byte ) => { binary += String . fromCharCode ( byte ); });
return `Basic ${ btoa ( binary ) } ` ;
}
function aiRequestHeaders ( extra = {}) {
return {
... extra ,
...( aiAuthorization ? { Authorization : aiAuthorization } : {}),
};
}
async function fetchIceServers ( baseUrl ) {
try {
const iceUrl = endpointUrl ( baseUrl , '/api/webrtc/ice-servers' );
const response = await fetch ( iceUrl , {
credentials : 'include' ,
headers : aiRequestHeaders (),
});
if ( ! response . ok ) throw new Error ( `HTTP ${ response . status } ` );
const payload = await response . json ();
if ( Array . isArray ( payload . iceServers )) return payload . iceServers ;
} catch ( error ) {
log ( `ICE server config unavailable, using public STUN: ${ error . message || error } ` );
}
return [{ urls : 'stun:stun.l.google.com:19302' }];
}
function queueAiIceCandidate ( candidate ) {
if ( ! candidate ) return ;
aiCandidateQueue . push ({
candidate : candidate . candidate ,
sdp_mid : candidate . sdpMid ,
sdp_mline_index : candidate . sdpMLineIndex ,
});
if ( ! aiCandidateFlushTimer ) {
aiCandidateFlushTimer = setTimeout ( flushAiIceCandidates , 200 );
}
}
async function flushAiIceCandidates () {
aiCandidateFlushTimer = null ;
if ( ! aiOfferUrl || ! aiPcId || ! aiCanSendCandidates || aiCandidateQueue . length === 0 ) return ;
const candidates = aiCandidateQueue . splice ( 0 , aiCandidateQueue . length );
try {
const response = await fetch ( aiOfferUrl , {
method : 'PATCH' ,
credentials : 'include' ,
headers : aiRequestHeaders ({ 'Content-Type' : 'application/json' }),
body : JSON . stringify ({ pc_id : aiPcId , candidates }),
});
if ( ! response . ok ) throw new Error ( `HTTP ${ response . status } ` );
} catch ( error ) {
aiCandidateQueue . unshift (... candidates );
log ( `failed to send ICE candidates: ${ error . message || error } ` );
}
}
function sendAiSignalling ( message ) {
if ( aiDc ? . readyState !== 'open' ) return ;
aiDc . send ( JSON . stringify ({ type : 'signalling' , message }));
}
async function negotiateAi ( restartPc = false ) {
if ( ! aiPc || ! aiOfferUrl || ! aiAssistantId || aiNegotiating ) return false ;
aiNegotiating = true ;
try {
const offer = await aiPc . createOffer ();
await aiPc . setLocalDescription ( offer );
const response = await fetch ( aiOfferUrl , {
method : 'POST' ,
credentials : 'include' ,
headers : aiRequestHeaders ({
'Content-Type' : 'application/json' ,
'X-Pipecat-Assistant-ID' : aiAssistantId ,
}),
body : JSON . stringify ({
pc_id : aiPcId ,
sdp : aiPc . localDescription . sdp ,
type : aiPc . localDescription . type ,
restart_pc : restartPc ,
requestData : {
assistant_id : aiAssistantId ,
vision_enabled : shouldSendVideoToAi (),
dynamic_variables : {},
},
}),
});
if ( ! response . ok ) {
const detail = await response . text (). catch (() => '' );
throw new Error ( `HTTP ${ response . status }${ detail ? `: ${ detail } ` : '' } ` );
}
const answer = await response . json ();
aiPcId = answer . pc_id ;
await aiPc . setRemoteDescription ({ type : answer . type , sdp : answer . sdp });
aiCanSendCandidates = true ;
await flushAiIceCandidates ();
log ( `AI WebRTC answer applied; ${ aiState () } ` , true );
return true ;
} finally {
aiNegotiating = false ;
}
}
async function handleAiSignalling ( message ) {
try {
if ( message ? . type === 'renegotiate' ) {
await negotiateAi ( false );
} else if ( message ? . type === 'peerLeft' ) {
setStatus ( 'disconnected' );
log ( 'AI peer left' , true );
} else {
log ( `unknown SmallWebRTC signalling: ${ message ? . type || 'missing type' } ` );
}
} catch ( error ) {
log ( `SmallWebRTC renegotiation failed: ${ error . message || error } ` , true );
}
}
async function connectAi ( baseUrl , assistantId , username , password ) {
2026-07-09 08:41:57 +08:00
scanMediaElements ();
selectRemoteAudio ( 'connect' );
selectRemoteVideo ( 'connect' );
selectBusinessSender ( 'connect' );
const status = gateStatus ();
updateGateLabel ();
2026-07-09 08:54:15 +08:00
if ( ! status . operatorOnline || ! status . participantOnline || ! status . senderReady ) {
2026-07-09 08:41:57 +08:00
appendChatLine ( '门槛' , gateMessage ( status ), true );
setStatus ( 'disconnected' );
return false ;
}
disconnectAi ();
selectedRemoteAudio = status . track ;
2026-07-13 12:12:04 +08:00
try {
aiOfferUrl = endpointUrl ( baseUrl , '/api/webrtc/offer' );
} catch {
appendChatLine ( '配置' , 'Base URL 格式无效,请填写包含 http:// 或 https:// 的完整地址' , true );
setStatus ( 'failed' );
return false ;
}
aiAssistantId = assistantId ;
aiAuthorization = basicAuthorization ( username , password );
aiPcId = pcId ();
aiCandidateQueue = [];
aiCanSendCandidates = false ;
const iceServers = await fetchIceServers ( baseUrl );
aiPc = new RTCPeerConnection ({ iceServers });
2026-07-09 08:41:57 +08:00
markInternalPc ( aiPc );
aiPc . onconnectionstatechange = () => log ( `AI PC connectionState= ${ aiPc . connectionState } ` );
aiPc . oniceconnectionstatechange = () => log ( `AI PC iceConnectionState= ${ aiPc . iceConnectionState } ` );
aiDc = aiPc . createDataChannel ( 'chat' );
aiDc . onopen = () => {
setStatus ( 'connected' );
log ( 'AI data channel open' , true );
2026-07-13 12:12:04 +08:00
sendAiSignalling ({ type : 'trackStatus' , receiver_index : 0 , enabled : shouldSendAudioToAi () });
sendAiSignalling ({ type : 'trackStatus' , receiver_index : 1 , enabled : shouldSendVideoToAi () });
2026-07-09 08:41:57 +08:00
aiDc . send ( JSON . stringify ({ type : 'client-ready' }));
2026-07-13 12:12:04 +08:00
aiKeepAliveTimer = setInterval (() => {
if ( aiDc ? . readyState === 'open' ) aiDc . send ( `ping: ${ Date . now () } ` );
}, 1000 );
2026-07-09 08:41:57 +08:00
};
aiDc . onclose = () => setStatus ( 'disconnected' );
aiDc . onmessage = ( event ) => {
try {
const msg = JSON . parse ( event . data );
const content = msg . delta || msg . content ;
2026-07-13 12:12:04 +08:00
if ( msg . type === 'signalling' ) {
void handleAiSignalling ( msg . message );
} else if ( msg . type === 'transcript' ) {
2026-07-09 08:41:57 +08:00
logUserTranscript ( content );
} else if ( msg . type === 'assistant-text-start' ) {
startAssistantLine ();
} else if ( msg . type === 'assistant-text-delta' ) {
appendAssistantDelta ( content );
} else if ( msg . type === 'assistant-text-end' ) {
finishAssistantLine ();
} else {
log ( `data channel message: ${ msg . type } ` );
}
} catch {
appendChatLine ( 'DATA' , event . data );
}
};
const sendAudio = shouldSendAudioToAi ();
2026-07-13 12:12:04 +08:00
const audioTransceiver = aiPc . addTransceiver ( 'audio' , { direction : 'sendrecv' });
2026-07-09 08:41:57 +08:00
aiAudioSender = audioTransceiver . sender ;
2026-07-13 12:12:04 +08:00
if ( sendAudio && isLive ( selectedRemoteAudio )) await aiAudioSender . replaceTrack ( selectedRemoteAudio );
2026-07-09 08:41:57 +08:00
log ( ! sendAudio
? 'AI audio uplink disabled by panel setting'
: isLive ( selectedRemoteAudio )
? `AI input starts with ${ trackLabel ( selectedRemoteAudio ) } ( ${ STATIC_AUDIO_ELEMENT_REASON } )`
: `AI input waiting for ${ STATIC_AUDIO_ELEMENT_REASON } ` , true );
const sendVideo = shouldSendVideoToAi ();
const shouldSendVideo = sendVideo && isLive ( selectedRemoteVideo ) && ! selectedRemoteVideo . muted ;
2026-07-13 12:12:04 +08:00
const videoTransceiver = aiPc . addTransceiver ( 'video' , { direction : 'sendrecv' });
aiVideoSender = videoTransceiver . sender ;
2026-07-09 08:41:57 +08:00
if ( shouldSendVideo ) {
2026-07-13 12:12:04 +08:00
await aiVideoSender . replaceTrack ( selectedRemoteVideo );
2026-07-09 08:41:57 +08:00
log ( `AI video starts with ${ trackLabel ( selectedRemoteVideo ) } ( ${ STATIC_VIDEO_ELEMENT_REASON } )` );
} else {
log ( sendVideo
? `AI video disabled: waiting for live track at ${ STATIC_VIDEO_ELEMENT_REASON } `
: 'AI video uplink disabled by panel setting' );
}
aiPc . ontrack = ( event ) => {
if ( event . track . kind !== 'audio' ) return ;
aiOutputTrackIds . add ( event . track . id );
remoteAudio . delete ( event . track . id );
aiAudioTrack = event . track ;
2026-07-09 08:54:15 +08:00
attachAiOutputMeter ( event . track );
2026-07-09 08:41:57 +08:00
log ( `received AI audio ${ trackLabel ( aiAudioTrack ) } ` , true );
aiAudioTrack . onunmute = () => {
log ( `AI audio unmuted ${ trackLabel ( aiAudioTrack ) } ` );
attachAiOutputToBusiness ();
};
attachAiOutputToBusiness ();
};
aiPc . onicecandidate = ( event ) => {
2026-07-13 12:12:04 +08:00
queueAiIceCandidate ( event . candidate );
2026-07-09 08:41:57 +08:00
};
setStatus ( 'connecting' );
2026-07-13 12:12:04 +08:00
try {
await negotiateAi ( false );
log ( 'sent SmallWebRTC HTTP offer' , true );
} catch ( error ) {
log ( `AI HTTP signalling error: ${ error . message || error } ` , true );
disconnectAi ();
2026-07-09 08:41:57 +08:00
setStatus ( 'failed' );
2026-07-13 12:12:04 +08:00
return false ;
}
2026-07-09 08:41:57 +08:00
attachAudioToAi ();
attachVideoToAi ();
return true ;
}
function disconnectAi () {
try { aiDc ? . close (); } catch {}
try { aiPc ? . close (); } catch {}
2026-07-13 12:12:04 +08:00
if ( aiCandidateFlushTimer ) clearTimeout ( aiCandidateFlushTimer );
if ( aiKeepAliveTimer ) clearInterval ( aiKeepAliveTimer );
2026-07-09 08:41:57 +08:00
aiDc = null ;
aiPc = null ;
2026-07-13 12:12:04 +08:00
aiPcId = null ;
aiOfferUrl = null ;
aiAssistantId = null ;
aiAuthorization = null ;
aiCandidateQueue = [];
aiCandidateFlushTimer = null ;
aiKeepAliveTimer = null ;
aiNegotiating = false ;
aiCanSendCandidates = false ;
2026-07-09 08:41:57 +08:00
aiAudioSender = null ;
aiVideoSender = null ;
aiAudioTrack = null ;
2026-07-09 08:54:15 +08:00
detachAiOutputMeter ();
2026-07-09 08:41:57 +08:00
if ( inputRecorder ? . state === 'recording' ) stopInputRecording ();
2026-07-13 12:12:04 +08:00
if ( senderRecorder ? . state === 'recording' ) senderRecorder . stop ();
2026-07-09 08:41:57 +08:00
if ( videoRecorder ? . state === 'recording' ) stopVideoRecording ();
setStatus ( 'disconnected' );
}
function setStatus ( status ) {
const el = document . getElementById ( 'element2-copilot-status' );
if ( ! el ) return ;
el . textContent = {
disconnected : '未连接' ,
connecting : '连接中' ,
connected : '已连接' ,
failed : '连接失败' ,
}[ status ] || status ;
}
function updateModeButton () {
const button = document . getElementById ( 'element2-copilot-toggle' );
if ( ! button ) return ;
if ( isAiTakeover ) {
button . textContent = '当前:AI 接管|点击切回人工' ;
button . style . background = '#8b5cf6' ;
} else {
button . textContent = '当前:人工接管|点击切换 AI' ;
button . style . background = '#10b981' ;
}
}
window . __aiElement2GatedDump = function () {
const status = gateStatus ();
return {
2026-07-13 12:12:04 +08:00
audioSelection : {
mode : audioSelectionMode ,
manualTrackId : manualAudioTrackId ,
preferredElement : STATIC_AUDIO_ELEMENT_REASON ,
},
videoSelection : {
mode : videoSelectionMode ,
manualTrackId : manualVideoTrackId ,
preferredElement : STATIC_VIDEO_ELEMENT_REASON ,
},
senderSelection : {
mode : senderSelectionMode ,
manualSenderId ,
},
2026-07-09 08:41:57 +08:00
staticVideoElement : STATIC_VIDEO_ELEMENT_REASON ,
gate : {
bothParties : status . bothParties ,
2026-07-09 08:54:15 +08:00
participantOnline : status . participantOnline ,
operatorOnline : status . operatorOnline ,
2026-07-09 08:41:57 +08:00
senderReady : status . senderReady ,
track : trackLabel ( status . track ),
},
gateMessage : gateMessage ( status ),
selectedInput : trackLabel ( selectedRemoteAudio ),
selectedVideo : trackLabel ( selectedRemoteVideo ),
aiVideoSenderTrack : trackLabel ( aiVideoSender ? . track ),
selectedOutput : trackLabel ( selectedBusinessSender ? . track ),
originalOutput : trackLabel ( originalBusinessTrack ),
aiAudio : trackLabel ( aiAudioTrack ),
aiAttached ,
aiState : aiState (),
recording : inputRecorder ? . state === 'recording' ? {
track : trackLabel ( inputRecordingTrack ),
seconds : (( Date . now () - inputRecordingStartedAt ) / 1000 ). toFixed ( 1 ),
chunks : inputRecordingChunks . length ,
} : null ,
2026-07-13 12:12:04 +08:00
senderRecording : senderRecorder ? . state === 'recording' ? {
track : trackLabel ( senderRecordingTrack ),
seconds : (( Date . now () - senderRecordingStartedAt ) / 1000 ). toFixed ( 1 ),
chunks : senderRecordingChunks . length ,
} : null ,
2026-07-09 08:41:57 +08:00
videoRecording : videoRecorder ? . state === 'recording' ? {
track : trackLabel ( videoRecordingTrack ),
seconds : (( Date . now () - videoRecordingStartedAt ) / 1000 ). toFixed ( 1 ),
chunks : videoRecordingChunks . length ,
} : null ,
inputs : Array . from ( remoteAudio . values ()). map (( item ) => ({
selected : item . track === selectedRemoteAudio ,
id : item . track . id ,
muted : item . track . muted ,
live : item . track . readyState ,
hasReceiver : Boolean ( item . receiver ),
source : audioSourceLabel ( item ),
bytesReceived : item . bytesReceived || 0 ,
packetsReceived : item . packetsReceived || 0 ,
bytesDelta : item . bytesDelta ,
packetsDelta : item . packetsDelta ,
audioLevel : item . audioLevel || 0 ,
audioScore : Math . round ( item . audioScore || scoreAudioCandidate ( item ) || 0 ),
matchesStaticElement : hasElementReason ( item , STATIC_AUDIO_ELEMENT_REASON ),
reasons : Array . from ( item . reasons ),
})),
videos : Array . from ( remoteVideo . values ()). map (( item ) => ({
selected : item . track === selectedRemoteVideo ,
id : item . track . id ,
muted : item . track . muted ,
live : item . track . readyState ,
hasReceiver : Boolean ( item . receiver ),
matchesStaticElement : hasElementReason ( item , STATIC_VIDEO_ELEMENT_REASON ),
reasons : Array . from ( item . reasons ),
})),
outputs : Array . from ( businessSenders . values ()). map (( item ) => ({
selected : item . sender === selectedBusinessSender ,
id : item . id ,
track : trackLabel ( item . sender . track || item . track ),
originalTrack : trackLabel ( item . originalTrack ),
pcAddTrack : item . sources . has ( 'pc-addTrack' ),
isRemoteInput : isRemoteInputTrack ( item . sender . track || item . track ),
bytesSent : item . bytesSent ,
packetsSent : item . packetsSent ,
bytesDelta : item . bytesDelta ,
packetsDelta : item . packetsDelta ,
sources : Array . from ( item . sources ),
})),
};
};
function renderPanel () {
if ( ! document . body || document . getElementById ( 'element2-copilot-panel' )) return ;
const panel = document . createElement ( 'div' );
panel . id = 'element2-copilot-panel' ;
panel . style . cssText = [
'position:fixed' ,
'top:20px' ,
'left:20px' ,
2026-07-13 12:12:04 +08:00
'width:440px' ,
'max-height:calc(100vh - 40px)' ,
'overflow:auto' ,
2026-07-09 08:41:57 +08:00
'z-index:999999' ,
'box-sizing:border-box' ,
'padding:16px' ,
'border-radius:10px' ,
'background:rgba(17,24,39,.96)' ,
'color:white' ,
'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif' ,
'box-shadow:0 12px 32px rgba(0,0,0,.35)' ,
]. join ( ';' );
panel . innerHTML = `
<div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
2026-07-09 08:54:15 +08:00
<strong style="font-size:15px;color:#f9a8d4;">AI 托管</strong>
2026-07-09 08:41:57 +08:00
<span id="element2-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
</div>
<div style="font-size:11px;color:#9ca3af;margin-bottom:10px;line-height:1.5;">
2026-07-09 08:54:15 +08:00
<span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人在线</span>
2026-07-09 08:41:57 +08:00
</div>
2026-07-13 12:12:04 +08:00
<div style="display:grid;grid-template-columns:1.35fr 1fr;gap:8px;margin-bottom:10px;">
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 服务 Base URL</label>
<input id="element2-copilot-url" value=" ${ DEFAULT_API_BASE_URL } " placeholder="例如:https://ai.example.com" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
</div>
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">助手 ID</label>
<input id="element2-copilot-assistant" placeholder="请输入助手 ID" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
</div>
2026-07-09 08:41:57 +08:00
</div>
2026-07-13 12:12:04 +08:00
<div style="display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-bottom:12px;">
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">管理员用户名</label>
<input id="element2-copilot-username" value="admin" autocomplete="username" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
</div>
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">管理员密码</label>
<input id="element2-copilot-password" type="password" autocomplete="current-password" placeholder="请输入密码" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
</div>
</div>
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;margin-bottom:9px;">
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 输入选轨</label>
<select id="element2-copilot-audio-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="auto">自动选轨</option>
<option value="manual">手动选轨</option>
<option value="off">不传音频</option>
</select>
</div>
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">远端音轨</label>
<select id="element2-copilot-audio-track" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="">暂无可用远端音轨</option>
</select>
</div>
<button id="element2-copilot-record" title="录制 AI 输入" aria-label="录制 AI 输入" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#2563eb;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
</div>
2026-07-09 08:54:15 +08:00
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">截取给 AI 的音频音量</span><span id="element2-copilot-input-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
2026-07-09 08:41:57 +08:00
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div></div>
</div>
2026-07-13 12:12:04 +08:00
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;margin-bottom:9px;">
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 输出 sender</label>
<select id="element2-copilot-sender-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="auto">自动选择</option>
<option value="manual">手动选择</option>
</select>
</div>
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">业务上行 sender</label>
<select id="element2-copilot-sender" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="">暂无可用 sender</option>
</select>
</div>
<button id="element2-copilot-record-sender" title="录制 sender 音轨" aria-label="录制 sender 音轨" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#7c3aed;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
</div>
2026-07-09 08:54:15 +08:00
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">AI 发送给当事人的音频音量</span><span id="element2-copilot-output-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
2026-07-09 08:41:57 +08:00
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-output-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div></div>
</div>
2026-07-13 12:12:04 +08:00
<div style="margin-bottom:12px;padding:10px;border:1px solid #374151;border-radius:8px;background:rgba(3,7,18,.45);">
<div style="display:grid;grid-template-columns:110px 1fr 34px;gap:8px;align-items:end;">
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">AI 视频选轨</label>
<select id="element2-copilot-video-mode" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="auto">自动选轨</option>
<option value="manual">手动选轨</option>
<option value="off">不传视频</option>
</select>
</div>
<div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">远端视频轨</label>
<select id="element2-copilot-video-track" style="width:100%;box-sizing:border-box;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
<option value="">暂无可用远端视频</option>
</select>
</div>
<button id="element2-copilot-record-video" title="录制 AI 视频" aria-label="录制 AI 视频" style="width:34px;height:34px;padding:0;border:0;border-radius:6px;background:#475569;color:white;font-size:13px;line-height:34px;cursor:pointer;">●</button>
</div>
2026-07-09 08:41:57 +08:00
</div>
2026-07-13 12:12:04 +08:00
<button id="element2-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前:人工接管|点击切换 AI</button>
2026-07-09 08:41:57 +08:00
<div id="element2-copilot-log" style="height:170px;margin-top:12px;padding:8px;overflow:auto;border:1px solid #374151;border-radius:6px;background:#030712;color:#a7f3d0;font:11px ui-monospace,Menlo,monospace;">
<span data-placeholder="1" style="color:#6b7280;">等待交互...</span>
</div>
` ;
document . body . appendChild ( panel );
const toggle = document . getElementById ( 'element2-copilot-toggle' );
const recordButton = document . getElementById ( 'element2-copilot-record' );
2026-07-13 12:12:04 +08:00
const senderRecordButton = document . getElementById ( 'element2-copilot-record-sender' );
2026-07-09 08:41:57 +08:00
const videoRecordButton = document . getElementById ( 'element2-copilot-record-video' );
const urlInput = document . getElementById ( 'element2-copilot-url' );
const assistantInput = document . getElementById ( 'element2-copilot-assistant' );
2026-07-13 12:12:04 +08:00
const usernameInput = document . getElementById ( 'element2-copilot-username' );
const passwordInput = document . getElementById ( 'element2-copilot-password' );
const audioModeInput = document . getElementById ( 'element2-copilot-audio-mode' );
const audioTrackInput = document . getElementById ( 'element2-copilot-audio-track' );
const videoModeInput = document . getElementById ( 'element2-copilot-video-mode' );
const videoTrackInput = document . getElementById ( 'element2-copilot-video-track' );
const senderModeInput = document . getElementById ( 'element2-copilot-sender-mode' );
const senderInput = document . getElementById ( 'element2-copilot-sender' );
audioModeInput . value = audioSelectionMode ;
audioModeInput . addEventListener ( 'change' , () => {
writeAudioSelectionMode ( audioModeInput . value );
if ( audioSelectionMode === 'manual' ) {
manualAudioTrackId = selectedRemoteAudio ? . id || selectableAudioItems ()[ 0 ] ? . track . id || null ;
}
selectRemoteAudio ( 'mode-change' );
log ( `AI input selection mode: ${ audioSelectionMode } ` );
2026-07-09 08:41:57 +08:00
});
2026-07-13 12:12:04 +08:00
audioTrackInput . addEventListener ( 'change' , () => {
manualAudioTrackId = audioTrackInput . value || null ;
selectRemoteAudio ( 'manual-select' );
log ( `manual AI input track: ${ manualAudioTrackId || 'none' } ` );
});
videoModeInput . value = videoSelectionMode ;
videoModeInput . addEventListener ( 'change' , () => {
writeVideoSelectionMode ( videoModeInput . value );
if ( videoSelectionMode === 'manual' ) {
manualVideoTrackId = selectedRemoteVideo ? . id || selectableVideoItems ()[ 0 ] ? . track . id || null ;
}
videoTrackSelectSignature = '' ;
selectRemoteVideo ( 'video-mode-change' );
log ( `AI video selection mode: ${ videoSelectionMode } ` );
});
videoTrackInput . addEventListener ( 'change' , () => {
manualVideoTrackId = videoTrackInput . value || null ;
selectRemoteVideo ( 'manual-video-select' );
log ( `manual AI video track: ${ manualVideoTrackId || 'none' } ` );
});
senderModeInput . value = senderSelectionMode ;
senderModeInput . addEventListener ( 'change' , async () => {
writeSenderSelectionMode ( senderModeInput . value );
if ( senderSelectionMode === 'manual' ) {
const selectedItem = Array . from ( businessSenders . values ()). find (( item ) => item . sender === selectedBusinessSender );
manualSenderId = selectedItem ? . id || selectableSenderItems ()[ 0 ] ? . id || null ;
}
await changeBusinessSender ( 'sender-mode-change' );
log ( `business sender selection mode: ${ senderSelectionMode } ` );
});
senderInput . addEventListener ( 'change' , async () => {
manualSenderId = senderInput . value || null ;
await changeBusinessSender ( 'manual-sender-select' );
log ( `manual business sender: ${ manualSenderId || 'none' } ` );
2026-07-09 08:41:57 +08:00
});
recordButton . addEventListener ( 'click' , toggleInputRecording );
2026-07-13 12:12:04 +08:00
senderRecordButton . addEventListener ( 'click' , toggleSenderRecording );
2026-07-09 08:41:57 +08:00
videoRecordButton . addEventListener ( 'click' , toggleVideoRecording );
toggle . addEventListener ( 'click' , async () => {
if ( ! isAiTakeover && ! assistantInput . value . trim ()) {
alert ( '请先填写助手 ID。' );
return ;
}
2026-07-13 12:12:04 +08:00
if ( ! isAiTakeover && ( ! usernameInput . value . trim () || ! passwordInput . value )) {
alert ( '请填写管理员用户名和密码。' );
return ;
}
2026-07-09 08:41:57 +08:00
isAiTakeover = ! isAiTakeover ;
updateModeButton ();
if ( isAiTakeover ) {
2026-07-13 12:12:04 +08:00
const connected = await connectAi (
urlInput . value . trim (),
assistantInput . value . trim (),
usernameInput . value . trim (),
passwordInput . value ,
);
2026-07-09 08:41:57 +08:00
if ( connected ) {
await attachAiOutputToBusiness ();
} else {
isAiTakeover = false ;
updateModeButton ();
}
} else {
await restoreBusinessAudio ();
disconnectAi ();
}
});
updateModeButton ();
updateRecordButton ();
2026-07-13 12:12:04 +08:00
updateSenderRecordButton ();
2026-07-09 08:41:57 +08:00
updateVideoRecordButton ();
2026-07-13 12:12:04 +08:00
selectRemoteAudio ( 'panel-ready' );
selectRemoteVideo ( 'panel-ready' );
selectBusinessSender ( 'panel-ready' );
2026-07-09 08:41:57 +08:00
updateGateLabel ();
}
function renderPanelSoon () {
try { renderPanel (); } catch ( error ) { console . warn ( '[AI Element2 Gated] panel error' , error ); }
}
if ( document . readyState === 'loading' ) {
document . addEventListener ( 'DOMContentLoaded' , renderPanelSoon , { once : true });
} else {
renderPanelSoon ();
}
requestAnimationFrame ( renderVu );
setInterval ( refreshStats , 250 );
setTimeout ( renderPanelSoon , 1500 );
setTimeout ( renderPanelSoon , 3000 );
window . addEventListener ( 'beforeunload' , () => {
if ( inputRecorder ? . state === 'recording' ) stopInputRecording ();
2026-07-13 12:12:04 +08:00
if ( senderRecorder ? . state === 'recording' ) senderRecorder . stop ();
2026-07-09 08:41:57 +08:00
if ( videoRecorder ? . state === 'recording' ) stopVideoRecording ();
});
})();