接入 LiveKit 与 TurnSense 评分并改进模型曲线选择
This commit is contained in:
Xin Wang
@@ -1,12 +1,14 @@
|
||||
"use client";
|
||||
|
||||
import { useEffect, useMemo, useState } from "react";
|
||||
import { useEffect, useMemo, useRef, useState, useId } from "react";
|
||||
import { scoreAt, scorePath, type ModelId, type ScoreResponse, type ScoreSeries } from "@/lib/model-scores";
|
||||
import { createPortal } from "react-dom";
|
||||
import { SCORE_MODELS, SCORE_MODEL_IDS, readScorePreferences } from "@/lib/score-models";
|
||||
import type { TimelineAnnotation } from "./timeline";
|
||||
import styles from "./timeline.module.css";
|
||||
|
||||
const EMPTY: ScoreSeries[] = [];
|
||||
const COLORS: Record<ModelId, string> = { maai: "#8866b2", smart_turn: "#278d95", silero_vad: "#c47b32" };
|
||||
const COLORS = Object.fromEntries(SCORE_MODEL_IDS.map(id => [id, SCORE_MODELS[id].color])) as Record<ModelId, string>;
|
||||
export const SCORE_HEIGHT = 172;
|
||||
export type ScoreThresholds = Record<ModelId, number>;
|
||||
export type ScoreVisibility = Record<ModelId, boolean>;
|
||||
@@ -44,33 +46,125 @@ export function useModelScores(callId: string, recordingPath: string | null, exp
|
||||
};
|
||||
}
|
||||
|
||||
export function ScoreControls({ series, currentMs, thresholds, onThreshold, visibility, onVisibility }: {
|
||||
export function useScorePreferences() {
|
||||
const [preferences, setPreferences] = useState(() => readScorePreferences(null));
|
||||
const [ready, setReady] = useState(false);
|
||||
useEffect(() => {
|
||||
try { setPreferences(readScorePreferences(JSON.parse(localStorage.getItem("eot-score-preferences-v1") || "null"))); } catch { /* Defaults survive unavailable or invalid browser storage. */ }
|
||||
setReady(true);
|
||||
}, []);
|
||||
useEffect(() => {
|
||||
if (!ready) return;
|
||||
try { localStorage.setItem("eot-score-preferences-v1", JSON.stringify({ version: 1, ...preferences })); } catch { /* Preferences are optional. */ }
|
||||
}, [ready, preferences]);
|
||||
return { ...preferences,
|
||||
setVisibility: (visibility: ScoreVisibility) => setPreferences(previous => ({ ...previous, visibility })),
|
||||
setThreshold: (id: ModelId, value: number) => setPreferences(previous => ({ ...previous, thresholds: { ...previous.thresholds, [id]: value } })),
|
||||
};
|
||||
}
|
||||
|
||||
export function ScoreControls({ series, currentMs, thresholds, onThreshold, visibility, onVisibility, onFocus, focusedModel, loading }: {
|
||||
series: ScoreSeries[]; currentMs: number; thresholds: ScoreThresholds;
|
||||
onThreshold: (model: ModelId, value: number) => void;
|
||||
visibility: ScoreVisibility; onVisibility: (model: ModelId, visible: boolean) => void;
|
||||
visibility: ScoreVisibility; onVisibility: (visibility: ScoreVisibility) => void;
|
||||
onFocus: (model: ModelId | null) => void; focusedModel: ModelId | null; loading: boolean;
|
||||
}) {
|
||||
const [panel, setPanel] = useState<ModelId | "choose" | null>(null);
|
||||
const [query, setQuery] = useState("");
|
||||
const [position, setPosition] = useState({ top: 0, left: 0 });
|
||||
const [previousSelection, setPreviousSelection] = useState<ScoreVisibility | null>(null);
|
||||
const anchor = useRef<HTMLButtonElement | null>(null);
|
||||
const popup = useRef<HTMLDivElement>(null);
|
||||
const search = useRef<HTMLInputElement>(null);
|
||||
const panelId = useId();
|
||||
const available = new Set(series.map(row => row.model));
|
||||
const selected = series.filter(row => visibility[row.model]);
|
||||
function close() { setPanel(null); anchor.current?.focus({ preventScroll: true }); }
|
||||
function open(kind: ModelId | "choose", button: HTMLButtonElement) {
|
||||
anchor.current = button;
|
||||
if (panel === kind) { close(); return; }
|
||||
setQuery(""); setPanel(kind);
|
||||
}
|
||||
useEffect(() => {
|
||||
if (!panel) return;
|
||||
const place = () => {
|
||||
const r = anchor.current?.getBoundingClientRect(); if (!r) return;
|
||||
const width = Math.min(360, window.innerWidth - 24);
|
||||
const height = Math.min(popup.current?.getBoundingClientRect().height || 460, window.innerHeight - 24);
|
||||
setPosition({ left: Math.max(12, Math.min(r.left, window.innerWidth - width - 12)),
|
||||
top: Math.max(12, Math.min(r.bottom + 8, window.innerHeight - height - 12)) });
|
||||
};
|
||||
place();
|
||||
(panel === "choose" ? search.current : popup.current?.querySelector<HTMLElement>("button,select"))?.focus({ preventScroll: true });
|
||||
const outside = (event: PointerEvent) => {
|
||||
if (!popup.current?.contains(event.target as Node) && !anchor.current?.contains(event.target as Node)) setPanel(null);
|
||||
};
|
||||
const escape = (event: KeyboardEvent) => {
|
||||
if (event.key === "Escape") { event.preventDefault(); event.stopPropagation(); close(); }
|
||||
// Keep workbench hotkeys from changing annotations while browsing settings.
|
||||
else if (popup.current?.contains(event.target as Node)) event.stopPropagation();
|
||||
};
|
||||
document.addEventListener("pointerdown", outside); document.addEventListener("keydown", escape, true);
|
||||
window.addEventListener("resize", place); window.addEventListener("scroll", place, true);
|
||||
return () => { document.removeEventListener("pointerdown", outside); document.removeEventListener("keydown", escape, true);
|
||||
window.removeEventListener("resize", place); window.removeEventListener("scroll", place, true); };
|
||||
}, [panel]);
|
||||
const manualSelection = (next: ScoreVisibility) => { setPreviousSelection(null); onVisibility(next); };
|
||||
const metadata = panel && panel !== "choose" ? SCORE_MODELS[panel] : null;
|
||||
return <div className={styles.scoreControls}>
|
||||
{series.map(row => {
|
||||
const visible = visibility[row.model];
|
||||
const point = visible ? scoreAt(row.points, currentMs, row.sample_interval_ms * 2) : null;
|
||||
return <div className={styles.modelControl} key={row.model} style={{ color: COLORS[row.model] }}>
|
||||
<button type="button" role="switch" aria-checked={visible} aria-label={`显示 ${row.name} 分数`}
|
||||
className={styles.modelVisibility} onClick={() => onVisibility(row.model, !visible)}>
|
||||
<span className={styles.modelSwitch} aria-hidden="true" /><strong>{row.name}</strong>
|
||||
</button>
|
||||
<output aria-live="off" title={!visible ? "已隐藏" : point ? `评分时刻 ${time(point[0])}` : "当前位置暂无评分"}>{point ? point[1].toFixed(3) : "—"}</output>
|
||||
<label>参考线<select aria-label={`${row.name} 参考阈值`} value={thresholds[row.model]} disabled={!visible}
|
||||
onChange={event => onThreshold(row.model, Number(event.target.value))}>
|
||||
{(row.model === "maai" ? [.11, .5, .8] : [.5, .8]).map(value => <option key={value} value={value}>{value.toFixed(2)}</option>)}
|
||||
<div className={styles.modelPickerBar}>
|
||||
<button type="button" className={styles.modelPickerButton} aria-expanded={panel === "choose"} aria-controls={panel === "choose" ? panelId : undefined}
|
||||
aria-haspopup="dialog" onClick={event => open("choose", event.currentTarget)}>选择模型 <span>{selected.length}/{SCORE_MODEL_IDS.length}</span><span aria-hidden="true">⌄</span></button>
|
||||
{previousSelection && <button type="button" className={styles.restoreSelection} onClick={() => { onVisibility(previousSelection); setPreviousSelection(null); }}>恢复之前选择</button>}
|
||||
</div>
|
||||
<div className={styles.modelLegend} aria-label="已显示模型图例">
|
||||
{selected.map(row => {
|
||||
const model = SCORE_MODELS[row.model];
|
||||
const point = scoreAt(row.points, currentMs, row.sample_interval_ms * 2);
|
||||
return <div key={row.model} className={styles.legendItem} style={{ color: model.color, opacity: focusedModel && focusedModel !== row.model ? .45 : 1 }}
|
||||
onPointerEnter={() => onFocus(row.model)} onPointerLeave={() => onFocus(null)}>
|
||||
<button type="button" className={styles.legendName} aria-label={`${model.name} 曲线设置`} aria-haspopup="dialog" aria-expanded={panel === row.model}
|
||||
onFocus={() => onFocus(row.model)} onBlur={() => onFocus(null)} onClick={event => open(row.model, event.currentTarget)}>
|
||||
<i className={styles.modelColor} /><span>{model.name}</span><output title={point ? `评分时刻 ${time(point[0])}` : "当前位置暂无评分"}>{point ? point[1].toFixed(3) : "—"}</output><span className={styles.legendMore} aria-hidden="true">⋯</span>
|
||||
</button>
|
||||
</div>;
|
||||
})}
|
||||
{!selected.length && <span className={styles.noModels}>{loading ? "正在载入评分…" : "尚未选择可用模型"}</span>}
|
||||
</div>
|
||||
{panel && createPortal(<div className={styles.modelPopover} ref={popup} id={panelId} role="dialog" aria-label={panel === "choose" ? "选择对比模型" : `${metadata!.name} 曲线设置`}
|
||||
style={{ top: position.top, left: position.left }}>
|
||||
<div className={styles.popoverHeading}><strong>{panel === "choose" ? "选择对比模型" : metadata!.name}</strong><button type="button" aria-label="关闭模型面板" onClick={close}>×</button></div>
|
||||
{panel === "choose" ? <>
|
||||
<input ref={search} className={styles.modelSearch} aria-label="搜索模型" placeholder="搜索模型名称或类型" value={query} onChange={event => setQuery(event.target.value)} />
|
||||
<div className={styles.pickerActions}><span>已显示 {selected.length} 个</span><button type="button" disabled={!series.length} onClick={() => manualSelection({ ...visibility, ...Object.fromEntries(series.map(row => [row.model, true])) })}>全选可用</button><button type="button" onClick={() => manualSelection(Object.fromEntries(SCORE_MODEL_IDS.map(id => [id, false])) as ScoreVisibility)}>清空</button></div>
|
||||
<div className={styles.modelOptions}>
|
||||
{Array.from(new Set(SCORE_MODEL_IDS.map(id => SCORE_MODELS[id].family))).reverse().map(family => {
|
||||
const models = SCORE_MODEL_IDS.filter(id => SCORE_MODELS[id].family === family && `${SCORE_MODELS[id].name} ${family}`.toLowerCase().includes(query.trim().toLowerCase()));
|
||||
return models.length ? <fieldset key={family}><legend>{family}</legend>{models.map(id => <label key={id} className={styles.modelOption}>
|
||||
<input type="checkbox" aria-label={`显示 ${SCORE_MODELS[id].name} 分数`} checked={visibility[id]} disabled={!available.has(id)} onChange={event => manualSelection({ ...visibility, [id]: event.target.checked })} />
|
||||
<i className={styles.modelColor} style={{ background: SCORE_MODELS[id].color }} /><span><strong>{SCORE_MODELS[id].name}</strong><small>{available.has(id) ? SCORE_MODELS[id].description.split("；")[0] : loading ? "正在读取评分…" : "当前录音暂无评分"}</small></span>
|
||||
</label>)}</fieldset> : null;
|
||||
})}
|
||||
{!SCORE_MODEL_IDS.some(id => `${SCORE_MODELS[id].name} ${SCORE_MODELS[id].family}`.toLowerCase().includes(query.trim().toLowerCase())) && <p className={styles.noSearchResults}>没有匹配的模型</p>}
|
||||
</div><p className={styles.pickerHint}>选择会自动记住。建议同时比较 2–3 条曲线。</p>
|
||||
</> : <>
|
||||
<p className={styles.modelExplanation}>{metadata!.description}</p>
|
||||
<label className={styles.thresholdSetting}>参考阈值<select aria-label={`${metadata!.name} 参考阈值`} value={thresholds[panel]} onChange={event => onThreshold(panel, Number(event.target.value))}>
|
||||
{metadata!.thresholds.map(value => <option key={value} value={value}>{value === .355 ? "0.355（中文）" : value.toFixed(2)}</option>)}
|
||||
</select></label>
|
||||
</div>;
|
||||
})}
|
||||
<button type="button" className={styles.modelMenuAction} onClick={() => {
|
||||
if (!previousSelection) setPreviousSelection(visibility);
|
||||
onVisibility(Object.fromEntries(SCORE_MODEL_IDS.map(id => [id, id === panel])) as ScoreVisibility); close();
|
||||
}}>仅显示此模型</button>
|
||||
<button type="button" className={styles.modelMenuAction} onClick={() => { manualSelection({ ...visibility, [panel]: false }); close(); }}>隐藏此模型</button>
|
||||
</>}
|
||||
</div>, document.body)}
|
||||
</div>;
|
||||
}
|
||||
|
||||
export default function ModelScoreTrack({ series, width, durationMs, currentMs, thresholds, annotations, onSeek, onSelectAnnotation }: {
|
||||
export default function ModelScoreTrack({ series, width, durationMs, currentMs, thresholds, annotations, onSeek, onSelectAnnotation, focusedModel }: {
|
||||
series: ScoreSeries[]; width: number; durationMs: number; currentMs: number;
|
||||
thresholds: ScoreThresholds; annotations: TimelineAnnotation[];
|
||||
thresholds: ScoreThresholds; annotations: TimelineAnnotation[]; focusedModel: ModelId | null;
|
||||
onSeek: (ms: number) => void; onSelectAnnotation: (id: string, seekToPoint?: boolean) => void;
|
||||
}) {
|
||||
const [hoverMs, setHoverMs] = useState<number | null>(null);
|
||||
@@ -86,7 +180,7 @@ export default function ModelScoreTrack({ series, width, durationMs, currentMs,
|
||||
};
|
||||
return <div className={styles.scoreTrack} onPointerLeave={() => setHoverMs(null)}>
|
||||
<svg width={width} height={SCORE_HEIGHT} viewBox={`0 0 ${width} ${SCORE_HEIGHT}`}
|
||||
role="group" aria-label="VAP、Smart Turn 与 Silero VAD 分数对比，纵轴 0 到 1，横轴与录音时间一致"
|
||||
role="group" aria-label="VAP、Smart Turn、Silero VAD、LiveKit v1-mini 与 TurnSense 分数对比，纵轴 0 到 1，横轴与录音时间一致"
|
||||
onPointerMove={event => {
|
||||
const bounds = event.currentTarget.getBoundingClientRect();
|
||||
setHoverMs(Math.max(0, Math.min(durationMs, (event.clientX - bounds.left) / bounds.width * durationMs)));
|
||||
@@ -94,10 +188,10 @@ export default function ModelScoreTrack({ series, width, durationMs, currentMs,
|
||||
{[0, .25, .5, .75, 1].map(p => <line key={p} x1={0} x2={width} y1={y(p)} y2={y(p)} stroke="#e8e9e4" strokeDasharray={p === 0 || p === 1 ? undefined : "2 4"} />)}
|
||||
{annotations.map(row => <line key={row.id} x1={x(row.decision_ms)} x2={x(row.decision_ms)} y1={0} y2={SCORE_HEIGHT}
|
||||
stroke={row.can_reply_now === "0" ? "#c68c6b" : "#96a781"} strokeDasharray="2 5" opacity=".4" />)}
|
||||
{paths.map(row => <g key={row.model}>
|
||||
{paths.map(row => <g key={row.model} data-model={row.model} opacity={focusedModel && focusedModel !== row.model ? .18 : 1}>
|
||||
<line x1={0} x2={width} y1={y(thresholds[row.model])} y2={y(thresholds[row.model])}
|
||||
stroke={COLORS[row.model]} strokeDasharray={row.model === "maai" ? "6 5" : "2 4"} opacity=".55" />
|
||||
<path d={row.path} fill="none" stroke={COLORS[row.model]} strokeWidth={1.6} strokeLinejoin="round" pointerEvents="none" />
|
||||
<path d={row.path} fill="none" stroke={COLORS[row.model]} strokeWidth={focusedModel === row.model ? 2.8 : 1.6} strokeLinejoin="round" pointerEvents="none" />
|
||||
{row.checkpoints.map(([ms, score]) => <circle key={ms} cx={x(ms)} cy={y(score)} r={3.8}
|
||||
fill="#fffefb" stroke={COLORS[row.model]} strokeWidth={1.8} tabIndex={0} role="button"
|
||||
aria-label={`${row.name} 原标注点评分 ${time(ms)}，${score.toFixed(3)}，点击定位`}
|
||||
|
||||
Reference in New Issue
Block a user