"""Analyze copied remote results against the existing local baselines.""" from collections import Counter import csv import hashlib import json from pathlib import Path HERE = Path(__file__).resolve().parent ROOT = HERE.parents[1] RESULTS = HERE/'results' def read(path): with path.open(encoding='utf-8-sig',newline='') as f: return list(csv.DictReader(f)) def metrics(rows, key): counts = Counter((int(r['can_reply_now']),int(r[key])) for r in rows) tn,fp,fn,tp = counts[0,0],counts[0,1],counts[1,0],counts[1,1] return dict(n=tn+fp+fn+tp,tn=tn,fp=fp,fn=fn,tp=tp, accuracy=(tn+tp)/len(rows),false_reply_rate=fp/(tn+fp),false_hold_rate=fn/(tp+fn)) def main(): labels_path = ROOT/'labels/EOT标注/工作数据/eot_labels.csv' label_hash = hashlib.sha256(labels_path.read_bytes()).hexdigest() soul = json.loads((RESULTS/'work-data-can-reply/summary.json').read_text()) soul_latency = json.loads((RESULTS/'latency.json').read_text()) maai_dir = ROOT/'experiments/maai-ch-kyoto/results' smart_dir = ROOT/'experiments/smart-turn-v3.2/results' maai = json.loads((maai_dir/'work-data-can-reply/summary.json').read_text()) smart = json.loads((smart_dir/'work-data-can-reply/summary.json').read_text()) assert soul['complete'] and soul['total_points']==44 and soul_latency['complete'] assert soul['metadata']['label_sha256']==maai['config']['label_sha256']==smart['config']['label_sha256']==label_hash for call,checksum in soul['audio_sha256'].items(): path=ROOT/'data/韩泰机器人抢话/音频'/f'{call}.mp3' assert hashlib.sha256(path.read_bytes()).hexdigest()==checksum==maai['config']['audio_sha256'][call] key=lambda r:(r['call_id'],r['decision_ms']) points=read(RESULTS/'work-data-can-reply/predictions.csv') smoke=read(RESULTS/'smoke/predictions.csv')[0] repeated=next(r for r in points if key(r)==key(smoke)) repeat_check=dict(key=list(key(smoke)), probability_difference=abs(float(smoke['probability'])-float(repeated['probability'])), same_asr=smoke['asr_text']==repeated['asr_text'], same_raw_state=smoke['raw_state']==repeated['raw_state']) assert repeat_check['probability_difference'] < 1e-6 and repeat_check['same_asr'], repeat_check m={key(r):r for r in read(maai_dir/'work-data-can-reply/predictions.csv')} s={key(r):r for r in read(smart_dir/'work-data-can-reply/predictions.csv')} labels=read(labels_path) assert len({key(r) for r in points})==44 assert set(map(key,points))==set(m)==set(s)==set(map(key,labels)) rows=[] frames={} for f in read(RESULTS/'work-data-can-reply/frame-times.csv'): frames.setdefault(key(f),[]).append(f) for r in points: a,b=m[key(r)],s[key(r)] assert r['can_reply_now']==a['can_reply_now']==b['can_reply_now'] history=frames[key(r)] assert len(history)==int(r['frame_count']) assert int(history[-1]['input_available_ms'])==int(r['decision_ms']) allow=0 last_state='initial_hold' last_ms=r['audio_start_ms'] for index,f in enumerate(history): assert int(f['frame'])==index and int(f['input_available_ms'])<=int(r['decision_ms']) if f['state']=='<|user_complete|>': allow=1;last_state=f['state'];last_ms=f['input_available_ms'] elif f['state'] in ('<|user_incomplete|>','<|user_nonidle|>'): allow=0;last_state=f['state'];last_ms=f['input_available_ms'] rows.append(dict(call_id=r['call_id'],decision_ms=r['decision_ms'], user_text_at_decision=r['user_text_at_decision'],can_reply_now=r['can_reply_now'], soulx_probability=float(r['probability']),soulx_05=int(float(r['probability'])>.5), soulx_08=int(float(r['probability'])>.8),soulx_raw_state=r['raw_state'], soulx_native_complete=int(r['raw_state']=='<|user_complete|>'),soulx_asr=r['asr_text'], soulx_corrected_complete=int(r['corrected_state']=='<|user_complete|>'), soulx_state_can_reply=allow,soulx_last_native_decision_state=last_state, soulx_last_native_decision_ms=last_ms, maai_probability=float(a['p_now_robot']),maai_05=int(float(a['p_now_robot'])>.5), maai_08=int(float(a['p_now_robot'])>.8), maai_turnbench_score=1-float(a['p_now_user']), maai_zero_false_hold=int(1-float(a['p_now_user'])>.11), smart_probability=float(b['probability']),smart_05=int(float(b['probability'])>.5), smart_08=int(float(b['probability'])>.8))) with (RESULTS/'comparison-points.csv').open('w',encoding='utf-8-sig',newline='') as f: w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows) metric={k:metrics(rows,k) for k in ('soulx_05','soulx_08','soulx_native_complete','soulx_corrected_complete','soulx_state_can_reply', 'maai_05','maai_08','maai_zero_false_hold','smart_05','smart_08')} assert metric['soulx_05']==soul['metrics'] for x in ('tn','fp','fn','tp','accuracy'): assert metric['maai_05'][x]==maai['main_metrics'][x] assert metric['smart_05'][x]==smart['metrics'][x] baseline_latency=json.loads((ROOT/'experiments/latency-comparison.json').read_text()) comparisons={base:dict( fixed=sum(r[base]!=int(r['can_reply_now']) and r['soulx_05']==int(r['can_reply_now']) for r in rows), regressed=sum(r[base]==int(r['can_reply_now']) and r['soulx_05']!=int(r['can_reply_now']) for r in rows)) for base in ('maai_05','smart_05','smart_08')} native_comparisons={base:dict( fixed=sum(r[base]!=int(r['can_reply_now']) and r['soulx_state_can_reply']==int(r['can_reply_now']) for r in rows), regressed=sum(r[base]==int(r['can_reply_now']) and r['soulx_state_can_reply']!=int(r['can_reply_now']) for r in rows)) for base in ('maai_05','smart_05','smart_08')} lowest_positive=min((r for r in rows if int(r['can_reply_now'])==1), key=lambda r:r['maai_turnbench_score']) zero_hold_scheme=dict(rule='1 - p_now[user] > 0.11',threshold=.11, metrics=metric['maai_zero_false_hold'],label_sha256=label_hash, selection='Supplementary operating point selected on these same 44 labels; not an independent validation or a guarantee of zero false holds on new audio.', lowest_positive=dict(call_id=lowest_positive['call_id'], decision_ms=lowest_positive['decision_ms'], score=lowest_positive['maai_turnbench_score'], text=lowest_positive['user_text_at_decision']), model_inference_unchanged=True, same_predictions_using_p_now_robot=all( r['maai_zero_false_hold']==int(r['maai_probability']>.11) for r in rows)) assert zero_hold_scheme['metrics']['fn']==0 assert zero_hold_scheme['same_predictions_using_p_now_robot'] (maai_dir/'work-data-can-reply/zero-false-hold.json').write_text( json.dumps(zero_hold_scheme,ensure_ascii=False,indent=2)+'\n') summary=dict(label_sha256=label_hash,metrics=metric,changed_vs_baseline=comparisons, maai_zero_false_hold_scheme=zero_hold_scheme, repeated_point_check=repeat_check, native_can_reply_mapping='Supplementary post-hoc mapping: start hold; complete -> allow; incomplete/nonidle -> hold; idle/backchannel retain previous value. No timeout, no service resets, no external EOT signals, no future frames.', native_can_reply_changed_vs_baseline=native_comparisons, soulx_native_state_metrics=soul['native_state_metrics'], soulx_corrected_state_metrics=soul['corrected_state_metrics'], soulx_threshold_diagnostics=soul['thresholds'], latency=dict(soulx=soul_latency,local_baselines=baseline_latency), method=soul['method'],limitations=soul['limitations']) (RESULTS/'comparison-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n') lines=['SoulX-Duplug / MaAI / Smart Turn：当前44个标注点对比', '', '数据：6段录音，can_reply=0/1各22点；标签和音频SHA256与本机基线一致。', '标签中29点来自xlsx草稿、15点没有明确审核信息，结果属于探索性标注点评估。', '', '准确率及误判（误放行：不该回复却允许；误阻止：可以回复却拦截）'] for name,k in [('SoulX 条件完成分数 >0.5（预先定义的分数口径）','soulx_05'),('SoulX 条件完成分数 >0.8','soulx_08'), ('SoulX 原生状态派生can_reply（辅助映射）','soulx_state_can_reply'), ('MaAI p_now_robot >0.5','maai_05'),('MaAI p_now_robot >0.8','maai_08'), ('MaAI 1-p_now_user >0.11（本批误阻止为0的补充方案）','maai_zero_false_hold'), ('Smart Turn >0.5','smart_05'),('Smart Turn >0.8','smart_08')]: x=metric[k] lines.append(f"{name}：{x['accuracy']:.1%}；误放行{x['fp']}/22，误阻止{x['fn']}/22；TN/FP/FN/TP={x['tn']}/{x['fp']}/{x['fn']}/{x['tp']}") lines.extend(['', 'MaAI补充方案：1-p_now_user >0.11；在本批44点中与p_now_robot >0.11的判断完全一致。', f"可回复点的最低分为{lowest_positive['maai_turnbench_score']:.9f}，来自{lowest_positive['call_id']} {lowest_positive['decision_ms']}ms：{lowest_positive['user_text_at_decision']}。", '阈值0.11在这批标签上选择，误阻止0/22、误放行17/22（77.27%），不能保证新数据仍零误阻止；模型推理及已测量计算延迟不变。', '在同为误阻止0的配置下，MaAI 0.11比Smart Turn 0.5少1个误放行，比Smart Turn 0.8多1个误放行。', '', 'SoulX原生五状态argmax恰为user_complete的诊断指标：'+json.dumps(metric['soulx_native_complete']), 'SoulX内部状态修正后恰为user_complete的诊断指标：'+json.dumps(metric['soulx_corrected_complete']), '主指标在每个判断点比较complete/incomplete logits，排除idle/nonidle等类别后归一化；分数不是官方校准的can_reply概率。', '原生状态诊断与主指标不是同一种决策，不能把主指标宣称为完整官方服务效果。', '补充原生状态映射：开始为不可回复；complete置为可回复，incomplete/nonidle置为不可回复，idle/backchannel保持前值。只使用该标注点之前的原生预测，不加入超时、外部结束信号或音频清空。', '这是为区分瞬时完成事件与持续可回复状态而增加的辅助映射，保留预先定义的条件分数结果，未按标签调参。', '只在complete/incomplete间归一化，会忽略idle/nonidle等类别，可能产生虚高分数；条件分数结果不能直接代表原生SoulX性能。', 'SoulX使用用户单声道及最多19.88秒真实音频；MaAI使用双声道及最多20秒；Smart Turn使用用户单声道及最多8秒。', 'SoulX每个判断点重新初始化KV，直接使用原生infer，保留内部ASR和状态计算，不使用超时回复、完成后的服务重置或最终整句ASR。', '最终处理块在判断点前40ms结束，前向音频在判断点结束；不读标注点后的音频，也不输入标签转写。', '已知cb3238dc 13540和e304f740 11531存在标注/ASR对齐问题，未改动人工标签。', '', '计算延迟（跨机器配置对比）']) for name,x in [('MaAI / M4 CPU4线程，每100ms增量',baseline_latency['maai_4_threads_incremental']), ('Smart Turn / M4 CPU4线程，8秒窗口',baseline_latency['smart_turn_4_threads']), ('Smart Turn / M4 CPU1线程，8秒窗口',baseline_latency['smart_turn_1_thread']), ('SoulX / RTX3090 + i5 CPU4线程，每160ms增量，含ASR',soul_latency['summary']['steady']['total_ms']), ('SoulX / 同上，扣除cascade ASR调用时间',soul_latency['summary']['steady']['non_cascade_ms'])]: lines.append(f"{name}：均值{x['mean_ms']:.2f}ms，P95 {x['p95_ms']:.2f}ms，样本数{x['n']}") lines.extend([f"SoulX全段计算RTF={soul_latency['realtime_factor']:.3f}，超过160ms更新周期的帧比例={soul_latency['fraction_over_160ms']:.1%}。", 'SoulX用CUDA同步后的墙钟计时，分解音频特征/分词器、ASR阶段和状态模型，模型加载与离线重放单独记录。', '扣除cascade ASR的数值仍包括ASR触发检测、文本处理等，不等同于纯状态网络时间。', 'MaAI是100ms双声道增量，Smart Turn每次计算8秒窗口，SoulX是160ms增量；处理量和硬件不同，不能据此断言同机速度排名。', 'Smart Turn包含约0.16ms重采样；MaAI和SoulX计时输入已为16kHz，重采样/文件解码不包含在增量计时内。', f"SoulX实际参数dtype={soul['metadata']['parameter_dtypes']}；官方配置写bf16但加载器没有转换，本次未添加自动混合精度。", f"SoulX点评估平均离线重放={sum(float(r['replay_inference_ms']) for r in points)/len(points):.2f}ms，不作为在线单步延迟。", f"SoulX热缓存加载耗时{soul['metadata']['load_ms']/1000:.2f}s（不包含Python导入），完整连续测试峰值GPU分配{soul_latency['peak_gpu_allocated_bytes']/2**30:.2f}GiB。", '160ms更新周期和40ms前向窗口没有计入计算耗时；约21.8%的帧计算时间超过160ms，应区分平均实时吞吐与局部排队延迟。', '原始最终实验日志有32次ASR list index out of range警告，官方适配器捕获后返回空文本；未过滤这些样本，日志保留以便进一步定位。', '', 'SoulX误判点（0.5）']) for r in rows: if r['soulx_05']!=int(r['can_reply_now']): lines.append(f"{r['call_id']} {r['decision_ms']}ms 标签={r['can_reply_now']} 预测={r['soulx_05']} p={r['soulx_probability']:.5f} 文本={r['user_text_at_decision']} ASR={r['soulx_asr']}") lines.extend(['','SoulX原生状态派生can_reply的误判点（辅助映射）']) for r in rows: if r['soulx_state_can_reply']!=int(r['can_reply_now']): lines.append(f"{r['call_id']} {r['decision_ms']}ms 标签={r['can_reply_now']} 预测={r['soulx_state_can_reply']} 最后决定状态={r['soulx_last_native_decision_state']} 文本={r['user_text_at_decision']} ASR={r['soulx_asr']}") lines.extend(['','本次数据的判断：', 'MaAI在相同误阻止数量4点下，比SoulX原生状态辅助映射少7个误放行（7 vs 14），整体准确率最高。', 'SoulX原生状态辅助映射相比Smart Turn 0.5少4个误放行，但多4个误阻止，准确率均为59.1%。', 'SoulX误放行主要仍是完整短语中的停顿；原生状态辅助映射的4个误阻止集中在贵州省遵义市仁怀市、我姓宋、辽宁葫芦岛、姓赵，涉及地点/姓名和两个已知对齐疑点。', '条件两类分数整体偏高，0.8阈值仅拦截正确2/22个不可回复点，不适合直接当作当前业务的can_reply概率。', '在本次默认FP32、当前音频及映射配置下，SoulX未显示质量或计算延迟优势；不能推断其他数据、完整官方策略或BF16优化的表现。']) (RESULTS/'对比分析.txt').write_text('\n'.join(lines)+'\n') print(json.dumps(dict(metrics=metric,changed_vs_baseline=comparisons, soulx_latency=soul_latency['summary']['steady']['total_ms']),ensure_ascii=False,indent=2)) if __name__=='__main__': main()