新增 Mimi 与仅用户 VAP 对比并统一流式评测

2026-10-28 22:44:36 +08:00
parent 19d4e3c767
commit 6785e3a07b
55 changed files with 6160 additions and 31 deletions
@@ -0,0 +1,169 @@
+"""Score VAP annotation points from already available continuous stream outputs.
+
+No model inference, state resets, interpolation, or changes to archived results.
+"""
+import argparse
+from bisect import bisect_right
+from collections import Counter
+from datetime import datetime, timezone
+import csv
+import hashlib
+import json
+import math
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]
+LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
+SOURCES = {
+    "maai": ("score-timeline.json", "cpc-stereo", "CPC 降噪双声道"),
+    "maai_user_only": ("user-only/score-timeline.json", "cpc-user-only", "CPC 降噪机器人静音"),
+    "maai_mimi_standard": ("mimi-standard/score-timeline.json", "mimi-standard", "Mimi 标准双声道"),
+    "maai_mimi_mc": ("mimi-mc/score-timeline.json", "mimi-mc", "Mimi 降噪双声道"),
+}
+
+
+def sha(path):
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def last_available(points, decision_ms, max_age_ms):
+    """The same inclusive past-only, bounded-age rule used by Studio scoreAt."""
+    index = bisect_right(points, decision_ms, key=lambda p: p[0]) - 1
+    if index < 0 or decision_ms - points[index][0] > max_age_ms:
+        return None
+    return points[index]
+
+
+def classification(rows, threshold=.5):
+    labeled = [r for r in rows if str(r['can_reply_now']) in ('0', '1')]
+    available = [r for r in labeled if r['status'] == 'available']
+    counts = Counter((int(r['can_reply_now']), int(float(r['score']) > threshold)) for r in available)
+    tn, fp, fn, tp = counts[0,0], counts[0,1], counts[1,0], counts[1,1]
+    n = tn + fp + fn + tp
+    ratio = lambda a,b: a/b if b else None
+    return dict(n=n, requested_labeled=len(labeled), unavailable_labeled=len(labeled)-n,
+                tn=tn, fp=fp, fn=fn, tp=tp, correct=tn+tp, accuracy=ratio(tn+tp,n),
+                false_reply_rate=ratio(fp,fp+tn), false_hold_rate=ratio(fn,fn+tp),
+                recall=ratio(tp,tp+fn), precision=ratio(tp,tp+fp), f1=ratio(2*tp,2*tp+fp+fn))
+
+
+def write_csv(path, rows):
+    if not rows:
+        return
+    with path.open('w', encoding='utf-8-sig', newline='') as f:
+        writer = csv.DictWriter(f, fieldnames=list(rows[0]))
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def refresh_streaming(model):
+    source_file, directory, name = SOURCES[model]
+    source_path = HERE / 'results' / source_file
+    source_hash, label_hash = sha(source_path), sha(LABELS)
+    source = json.loads(source_path.read_text())
+    if source['model'] != model or source['schema_version'] != 1:
+        raise ValueError('Unexpected source export')
+    interval = source['sample_interval_ms']
+    assert math.isfinite(interval) and interval > 0
+    with LABELS.open(encoding='utf-8-sig', newline='') as f:
+        labels = list(csv.DictReader(f))
+    assert len({(r['call_id'], int(r['decision_ms'])) for r in labels}) == len(labels)
+    calls = {c['call_id']: c for c in source['calls']}
+    assert len(calls) == len(source['calls'])
+    assert {r['call_id'] for r in labels} <= calls.keys(), 'Missing annotated calls'
+    rows, exported_calls = [], []
+    for call_id, call in calls.items():
+        audio = ROOT / 'data' / call['audio_path']
+        assert audio.resolve().is_relative_to((ROOT/'data').resolve()), 'Audio outside project data'
+        assert sha(audio) == call['audio_sha256'], 'Source recording changed; rerun inference'
+        points = call['points']
+        assert all(len(p)==2 and math.isfinite(p[0]) and math.isfinite(p[1]) and
+                   0 <= p[0] <= call['duration_ms'] and 0 <= p[1] <= 1 for p in points)
+        assert all(a[0] < b[0] for a,b in zip(points,points[1:]))
+        checkpoints, sources = [], []
+        for label in sorted([r for r in labels if r['call_id']==call_id], key=lambda r:int(r['decision_ms'])):
+            decision = int(label['decision_ms'])
+            assert 0 <= decision <= call['duration_ms'], 'Annotation outside recording'
+            point = last_available(points, decision, interval*2)
+            row = dict(label)
+            row.update(status='available' if point is not None else 'unavailable',
+                       score_time_ms=point[0] if point is not None else '',
+                       score_age_ms=decision-point[0] if point is not None else '',
+                       score=point[1] if point is not None else '',
+                       prediction_05=int(point[1]>.5) if point is not None else '')
+            rows.append(row)
+            if point is not None:
+                checkpoints.append([decision, point[1]])
+                sources.append([decision, point[0]])
+        exported_calls.append(dict(call_id=call_id, audio_path=call['audio_path'], audio_sha256=call['audio_sha256'],
+                                   duration_ms=call['duration_ms'], points=points,
+                                   checkpoints=checkpoints, checkpoint_sources=sources))
+    provenance = dict(source_export=str(source_path.relative_to(ROOT)), source_export_sha256=source_hash,
+                      source_inference_provenance=source.get('provenance', {}),
+                      source_model_sha256=source.get('model_sha256', {}),
+                      label_sha256=label_hash, evaluation_script_sha256=sha(Path(__file__)))
+    method = ('Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with '
+              'availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. '
+              'No state reset, future score, interpolation or independent-window checkpoint. '
+              'Availability includes source resampler buffering; score is the stored activity-projection proxy.')
+    exported = dict(schema_version=1, model=model, name=source['name'], score=source['score'],
+                    checkpoint_mode='streaming', sample_interval_ms=interval, method=method,
+                    max_score_age_ms=interval*2, provenance=provenance, calls=exported_calls)
+    summary = dict(model=model, name=name, evaluation_mode='streaming',
+                   created_at=datetime.now(timezone.utc).isoformat(), threshold=.5,
+                   rule='1 - p_now[user] > 0.5', main_metrics=classification(rows),
+                   thresholds=[dict(threshold=t, **classification(rows,t)) for t in (.11,.5,.7,.8,.9,.95,.98)],
+                   per_call={c:classification([r for r in rows if r['call_id']==c]) for c in calls},
+                   method=method, provenance=provenance,
+                   limitations=['Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.',
+                                'Uses stored rounded continuous scores; no new inference or model-speed measurement.',
+                                'No prior output or stale output is unavailable, not a negative/zero prediction.',
+                                'Archived independent-window results are a separate evaluation protocol.'])
+    assert sha(source_path)==source_hash and sha(LABELS)==label_hash, 'Inputs changed during evaluation'
+    output = HERE/'results/streaming'/directory
+    output.mkdir(parents=True, exist_ok=True)
+    write_csv(output/'predictions.csv', rows)
+    (output/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n')
+    (output/'score-timeline.json').write_text(json.dumps(exported,ensure_ascii=False,separators=(',',':'))+'\n')
+    return summary, rows
+
+
+def main(selected):
+    summaries, rows_by_model = {}, {}
+    for model in SOURCES if selected=='all' else (selected,):
+        summary, rows = refresh_streaming(model)
+        summaries[model], rows_by_model[model] = summary, rows
+        print(summary['name'], json.dumps(summary['main_metrics'],ensure_ascii=False),flush=True)
+    if selected != 'all':
+        return
+    label_hashes = {s['provenance']['label_sha256'] for s in summaries.values()}
+    assert len(label_hashes)==1, 'Models use different labels'
+    base = summaries['maai']['main_metrics']['accuracy']
+    for summary in summaries.values():
+        accuracy = summary['main_metrics']['accuracy']
+        summary['accuracy_delta_pp_vs_cpc_stereo'] = (accuracy-base)*100 if accuracy is not None and base is not None else None
+        summary['relative_accuracy_change_vs_cpc_stereo'] = (accuracy-base)/base if accuracy is not None and base else None
+    output = HERE/'results/streaming'
+    result = dict(evaluation_mode='streaming', label_sha256=next(iter(label_hashes)), models=summaries)
+    (output/'comparison-summary.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
+    lines = ['VAP 流式场景判断点对比（阈值严格 > 0.5）', '',
+             '模型 | 有效点/有标签点 | 正确数 | 准确率 | 误放行 FP | 误阻止 FN | 对 CPC 双声道变化']
+    for model, summary in summaries.items():
+        m = summary['main_metrics']
+        accuracy = f"{m['accuracy']:.2%}" if m['accuracy'] is not None else '不可用'
+        delta = summary['accuracy_delta_pp_vs_cpc_stereo']
+        delta_text = f'{delta:+.2f} 个百分点' if delta is not None else '不可用'
+        lines.append(f"{summary['name']} | {m['n']}/{m['requested_labeled']} | {m['correct']} | {accuracy} | {m['fp']} | {m['fn']} | {delta_text}")
+    lines += ['', '1. 本表使用连续推理在判断时刻已经产生的最新分数，与前端 VAP 圆点和悬停分数一致；不使用独立窗口结果。',
+              '2. 每个有效点记录 decision_ms、score_time_ms、score_age_ms；启动期或超过两倍更新间隔的旧分数不计为 0，而是不可用。',
+              '3. 数据为六通录音中的选定点，只做描述性比较；下一步宜扩充按通话隔离的测试集，再在验证集上选择阈值。',
+              '', '复现：在 MaAI 独立 uv 环境运行 python evaluate_streaming.py all。原独立窗口结果保留在原目录，另作历史诊断。']
+    (output/'对比分析.txt').write_text('\n'.join(lines)+'\n')
+    print('\n'.join(lines))
+
+
+if __name__=='__main__':
+    parser=argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('model',choices=('all',*SOURCES),default='all',nargs='?')
+    main(parser.parse_args().model)