93 lines
5.9 KiB
Python
93 lines
5.9 KiB
Python
"""Compare exact current points across Mimi standard/MC and CPC MC controls."""
|
||
from collections import Counter
|
||
import csv
|
||
import hashlib
|
||
import json
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
ROOT = HERE.parents[1]
|
||
OUT = HERE / "results/mimi-comparison"
|
||
|
||
|
||
def sha(path):
|
||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
||
|
||
def read(path):
|
||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||
return list(csv.DictReader(f))
|
||
|
||
|
||
def metrics(rows, field):
|
||
c = Counter((int(r['can_reply_now']), int(float(r[field]) > .5)) for r in rows)
|
||
tn, fp, fn, tp = c[0,0], c[0,1], c[1,0], c[1,1]
|
||
return dict(n=len(rows), correct=tn+tp, tn=tn, fp=fp, fn=fn, tp=tp,
|
||
accuracy=(tn+tp)/len(rows), false_reply_rate=fp/(fp+tn), false_hold_rate=fn/(fn+tp))
|
||
|
||
|
||
def main():
|
||
dirs = ['mimi-standard', 'mimi-mc', 'user-only']
|
||
labels = ROOT / 'labels/EOT标注/工作数据/eot_labels.csv'
|
||
summaries = {d: json.loads((HERE/'results'/d/'summary.json').read_text()) for d in dirs}
|
||
points = {d: {(r['call_id'], int(r['decision_ms'])):r for r in read(HERE/'results'/d/'predictions.csv')} for d in dirs}
|
||
current = {(r['call_id'], int(r['decision_ms'])):r['can_reply_now'] for r in read(labels) if r['can_reply_now'] in ('0','1')}
|
||
assert current and all(set(p)==set(current) for p in points.values())
|
||
assert all(s['provenance']['label_sha256']==sha(labels) for s in summaries.values())
|
||
assert summaries['mimi-standard']['provenance']['mc'] is False
|
||
assert summaries['mimi-mc']['provenance']['mc'] is True
|
||
audio_hashes = summaries['mimi-standard']['provenance']['audio_sha256']
|
||
assert all(s['provenance']['audio_sha256']==audio_hashes for s in summaries.values())
|
||
rows = []
|
||
for key, label in sorted(current.items()):
|
||
standard, mc, cpc = [points[d][key] for d in dirs]
|
||
assert all(r['can_reply_now']==label for r in (standard,mc,cpc))
|
||
assert standard['cpc_mc_stereo_score']==mc['cpc_mc_stereo_score']==cpc['stereo_can_reply_score']
|
||
rows.append(dict(call_id=key[0],decision_ms=key[1],can_reply_now=label,
|
||
mimi_standard=float(standard['can_reply_score']),mimi_mc=float(mc['can_reply_score']),
|
||
cpc_mc_stereo=float(cpc['stereo_can_reply_score']),cpc_mc_user_only=float(cpc['can_reply_score'])))
|
||
models = {
|
||
'cpc_mc_stereo':'CPC 降噪版（双声道）',
|
||
'cpc_mc_user_only':'CPC 降噪版（机器人静音）',
|
||
'mimi_standard':'Mimi 标准版（双声道）',
|
||
'mimi_mc':'Mimi 降噪版（双声道）',
|
||
}
|
||
result = {key:metrics(rows,key) for key in models}
|
||
base = result['cpc_mc_stereo']['accuracy']
|
||
for value in result.values():
|
||
value['accuracy_delta_pp_vs_cpc_mc_stereo'] = (value['accuracy']-base)*100
|
||
value['relative_accuracy_change_vs_cpc_mc_stereo'] = (value['accuracy']-base)/base
|
||
paired = {}
|
||
for key in ('mimi_standard','mimi_mc'):
|
||
paired[key] = dict(
|
||
fixed=sum((r['cpc_mc_stereo']>.5)!=bool(int(r['can_reply_now'])) and (r[key]>.5)==bool(int(r['can_reply_now'])) for r in rows),
|
||
regressed=sum((r['cpc_mc_stereo']>.5)==bool(int(r['can_reply_now'])) and (r[key]>.5)!=bool(int(r['can_reply_now'])) for r in rows))
|
||
summary = dict(threshold=.5,rule='1 - p_now[user] > 0.5',n=len(rows),positive=sum(r['can_reply_now']=='1' for r in rows),
|
||
models=result,paired_vs_cpc_mc_stereo=paired,
|
||
per_call={c:{m:metrics([r for r in rows if r['call_id']==c],m) for m in models} for c in sorted(audio_hashes)},
|
||
sources={d:{'summary_sha256':sha(HERE/'results'/d/'summary.json'),'predictions_sha256':sha(HERE/'results'/d/'predictions.csv')} for d in dirs},
|
||
label_sha256=sha(labels),audio_sha256=audio_hashes,
|
||
limitations=['Single seed 42 with exact-repeat witness, not multiple-seed statistics.',
|
||
'Only 44 selected points in six calls; descriptive comparison, not a held-out generalization claim.',
|
||
'Mimi standard/MC share the same encoder and framing; CPC uses a different encoder and native 100ms framing.',
|
||
'All scores are activity-projection mappings; no endpointing-policy or full-dialogue cutoff/latency claims.'])
|
||
OUT.mkdir(parents=True,exist_ok=True)
|
||
with (OUT/'comparison-points.csv').open('w',encoding='utf-8-sig',newline='') as f:
|
||
writer=csv.DictWriter(f,fieldnames=list(rows[0]));writer.writeheader();writer.writerows(rows)
|
||
(OUT/'comparison-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n')
|
||
text=['MaAI Mimi 与 CPC 对比（相同当前 44 点，阈值严格 > 0.5）','',
|
||
'模型 | 正确数 | 准确率 | 误放行 FP/22 | 误阻止 FN/22 | 相对 CPC 双声道准确率变化']
|
||
for key,name in models.items():
|
||
m=result[key]
|
||
text.append(f"{name} | {m['correct']}/{m['n']} | {m['accuracy']:.2%} | {m['fp']}/22 | {m['fn']}/22 | {m['accuracy_delta_pp_vs_cpc_mc_stereo']:+.2f} 个百分点（相对 {m['relative_accuracy_change_vs_cpc_mc_stereo']:+.2%}）")
|
||
text+=['','1. 标准版与降噪版的结果差异可直接从上表查看；二者使用相同 Mimi 编码器、双声道输入及 80 ms 帧，变化来自所选 VAP 权重。',
|
||
'2. 与 CPC 比较时同时存在编码器、checkpoint 和原生帧率差异；不把结果归结为 Mimi 编码器的单独因果效果。',
|
||
'3. 固定种子重复结果一致，但没有多种子误差条；六通录音的 44 个选定点不足以证明泛化优劣。',
|
||
'', '建议下一步：扩充并核验按通话隔离的测试集，在独立验证集上选择阈值；若要单独研究编码器，补充标准/降噪与 CPC/Mimi 的完整交叉配置。']
|
||
(OUT/'对比分析.txt').write_text('\n'.join(text)+'\n')
|
||
print('\n'.join(text))
|
||
|
||
|
||
if __name__=='__main__':
|
||
main()
|