"""Score VAP annotation points from already available continuous stream outputs. No model inference, state resets, interpolation, or changes to archived results. """ import argparse from bisect import bisect_right from collections import Counter from datetime import datetime, timezone import csv import hashlib import json import math from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv" SOURCES = { "maai": ("score-timeline.json", "cpc-stereo", "CPC 降噪双声道"), "maai_user_only": ("user-only/score-timeline.json", "cpc-user-only", "CPC 降噪机器人静音"), "maai_mimi_standard": ("mimi-standard/score-timeline.json", "mimi-standard", "Mimi 标准双声道"), "maai_mimi_mc": ("mimi-mc/score-timeline.json", "mimi-mc", "Mimi 降噪双声道"), } def sha(path): return hashlib.sha256(path.read_bytes()).hexdigest() def last_available(points, decision_ms, max_age_ms): """The same inclusive past-only, bounded-age rule used by Studio scoreAt.""" index = bisect_right(points, decision_ms, key=lambda p: p[0]) - 1 if index < 0 or decision_ms - points[index][0] > max_age_ms: return None return points[index] def classification(rows, threshold=.5): labeled = [r for r in rows if str(r['can_reply_now']) in ('0', '1')] available = [r for r in labeled if r['status'] == 'available'] counts = Counter((int(r['can_reply_now']), int(float(r['score']) > threshold)) for r in available) tn, fp, fn, tp = counts[0,0], counts[0,1], counts[1,0], counts[1,1] n = tn + fp + fn + tp ratio = lambda a,b: a/b if b else None return dict(n=n, requested_labeled=len(labeled), unavailable_labeled=len(labeled)-n, tn=tn, fp=fp, fn=fn, tp=tp, correct=tn+tp, accuracy=ratio(tn+tp,n), false_reply_rate=ratio(fp,fp+tn), false_hold_rate=ratio(fn,fn+tp), recall=ratio(tp,tp+fn), precision=ratio(tp,tp+fp), f1=ratio(2*tp,2*tp+fp+fn)) def write_csv(path, rows): if not rows: return with path.open('w', encoding='utf-8-sig', newline='') as f: writer = csv.DictWriter(f, fieldnames=list(rows[0])) writer.writeheader() writer.writerows(rows) def refresh_streaming(model): source_file, directory, name = SOURCES[model] source_path = HERE / 'results' / source_file source_hash, label_hash = sha(source_path), sha(LABELS) source = json.loads(source_path.read_text()) if source['model'] != model or source['schema_version'] != 1: raise ValueError('Unexpected source export') interval = source['sample_interval_ms'] assert math.isfinite(interval) and interval > 0 with LABELS.open(encoding='utf-8-sig', newline='') as f: labels = list(csv.DictReader(f)) assert len({(r['call_id'], int(r['decision_ms'])) for r in labels}) == len(labels) calls = {c['call_id']: c for c in source['calls']} assert len(calls) == len(source['calls']) assert {r['call_id'] for r in labels} <= calls.keys(), 'Missing annotated calls' rows, exported_calls = [], [] for call_id, call in calls.items(): audio = ROOT / 'data' / call['audio_path'] assert audio.resolve().is_relative_to((ROOT/'data').resolve()), 'Audio outside project data' assert sha(audio) == call['audio_sha256'], 'Source recording changed; rerun inference' points = call['points'] assert all(len(p)==2 and math.isfinite(p[0]) and math.isfinite(p[1]) and 0 <= p[0] <= call['duration_ms'] and 0 <= p[1] <= 1 for p in points) assert all(a[0] < b[0] for a,b in zip(points,points[1:])) checkpoints, sources = [], [] for label in sorted([r for r in labels if r['call_id']==call_id], key=lambda r:int(r['decision_ms'])): decision = int(label['decision_ms']) assert 0 <= decision <= call['duration_ms'], 'Annotation outside recording' point = last_available(points, decision, interval*2) row = dict(label) row.update(status='available' if point is not None else 'unavailable', score_time_ms=point[0] if point is not None else '', score_age_ms=decision-point[0] if point is not None else '', score=point[1] if point is not None else '', prediction_05=int(point[1]>.5) if point is not None else '') rows.append(row) if point is not None: checkpoints.append([decision, point[1]]) sources.append([decision, point[0]]) exported_calls.append(dict(call_id=call_id, audio_path=call['audio_path'], audio_sha256=call['audio_sha256'], duration_ms=call['duration_ms'], points=points, checkpoints=checkpoints, checkpoint_sources=sources)) provenance = dict(source_export=str(source_path.relative_to(ROOT)), source_export_sha256=source_hash, source_inference_provenance=source.get('provenance', {}), source_model_sha256=source.get('model_sha256', {}), label_sha256=label_hash, evaluation_script_sha256=sha(Path(__file__))) method = ('Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with ' 'availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. ' 'No state reset, future score, interpolation or independent-window checkpoint. ' 'Availability includes source resampler buffering; score is the stored activity-projection proxy.') exported = dict(schema_version=1, model=model, name=source['name'], score=source['score'], checkpoint_mode='streaming', sample_interval_ms=interval, method=method, max_score_age_ms=interval*2, provenance=provenance, calls=exported_calls) summary = dict(model=model, name=name, evaluation_mode='streaming', created_at=datetime.now(timezone.utc).isoformat(), threshold=.5, rule='1 - p_now[user] > 0.5', main_metrics=classification(rows), thresholds=[dict(threshold=t, **classification(rows,t)) for t in (.11,.5,.7,.8,.9,.95,.98)], per_call={c:classification([r for r in rows if r['call_id']==c]) for c in calls}, method=method, provenance=provenance, limitations=['Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.', 'Uses stored rounded continuous scores; no new inference or model-speed measurement.', 'No prior output or stale output is unavailable, not a negative/zero prediction.', 'Archived independent-window results are a separate evaluation protocol.']) assert sha(source_path)==source_hash and sha(LABELS)==label_hash, 'Inputs changed during evaluation' output = HERE/'results/streaming'/directory output.mkdir(parents=True, exist_ok=True) write_csv(output/'predictions.csv', rows) (output/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n') (output/'score-timeline.json').write_text(json.dumps(exported,ensure_ascii=False,separators=(',',':'))+'\n') return summary, rows def main(selected): summaries, rows_by_model = {}, {} for model in SOURCES if selected=='all' else (selected,): summary, rows = refresh_streaming(model) summaries[model], rows_by_model[model] = summary, rows print(summary['name'], json.dumps(summary['main_metrics'],ensure_ascii=False),flush=True) if selected != 'all': return label_hashes = {s['provenance']['label_sha256'] for s in summaries.values()} assert len(label_hashes)==1, 'Models use different labels' base = summaries['maai']['main_metrics']['accuracy'] for summary in summaries.values(): accuracy = summary['main_metrics']['accuracy'] summary['accuracy_delta_pp_vs_cpc_stereo'] = (accuracy-base)*100 if accuracy is not None and base is not None else None summary['relative_accuracy_change_vs_cpc_stereo'] = (accuracy-base)/base if accuracy is not None and base else None output = HERE/'results/streaming' result = dict(evaluation_mode='streaming', label_sha256=next(iter(label_hashes)), models=summaries) (output/'comparison-summary.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n') lines = ['VAP 流式场景判断点对比（阈值严格 > 0.5）', '', '模型 | 有效点/有标签点 | 正确数 | 准确率 | 误放行 FP | 误阻止 FN | 对 CPC 双声道变化'] for model, summary in summaries.items(): m = summary['main_metrics'] accuracy = f"{m['accuracy']:.2%}" if m['accuracy'] is not None else '不可用' delta = summary['accuracy_delta_pp_vs_cpc_stereo'] delta_text = f'{delta:+.2f} 个百分点' if delta is not None else '不可用' lines.append(f"{summary['name']} | {m['n']}/{m['requested_labeled']} | {m['correct']} | {accuracy} | {m['fp']} | {m['fn']} | {delta_text}") lines += ['', '1. 本表使用连续推理在判断时刻已经产生的最新分数，与前端 VAP 圆点和悬停分数一致；不使用独立窗口结果。', '2. 每个有效点记录 decision_ms、score_time_ms、score_age_ms；启动期或超过两倍更新间隔的旧分数不计为 0，而是不可用。', '3. 数据为六通录音中的选定点，只做描述性比较；下一步宜扩充按通话隔离的测试集，再在验证集上选择阈值。', '', '复现：在 MaAI 独立 uv 环境运行 python evaluate_streaming.py all。原独立窗口结果保留在原目录，另作历史诊断。'] (output/'对比分析.txt').write_text('\n'.join(lines)+'\n') print('\n'.join(lines)) if __name__=='__main__': parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('model',choices=('all',*SOURCES),default='all',nargs='?') main(parser.parse_args().model)