195 lines
15 KiB
Python
195 lines
15 KiB
Python
|
"""Analyze copied remote results against the existing local baselines."""
|
|||
|
|
from collections import Counter
|
|||
|
|
import csv
|
|||
|
|
import hashlib
|
|||
|
|
import json
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
HERE = Path(__file__).resolve().parent
|
|||
|
|
ROOT = HERE.parents[1]
|
|||
|
|
RESULTS = HERE/'results'
|
|||
|
|
|
|||
|
|
|
|||
|
|
def read(path):
|
|||
|
|
with path.open(encoding='utf-8-sig',newline='') as f:
|
|||
|
|
return list(csv.DictReader(f))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def metrics(rows, key):
|
|||
|
|
counts = Counter((int(r['can_reply_now']),int(r[key])) for r in rows)
|
|||
|
|
tn,fp,fn,tp = counts[0,0],counts[0,1],counts[1,0],counts[1,1]
|
|||
|
|
return dict(n=tn+fp+fn+tp,tn=tn,fp=fp,fn=fn,tp=tp,
|
|||
|
|
accuracy=(tn+tp)/len(rows),false_reply_rate=fp/(tn+fp),false_hold_rate=fn/(tp+fn))
|
|||
|
|
|
|||
|
|
|
|||
|
|
def main():
|
|||
|
|
labels_path = ROOT/'labels/EOT标注/工作数据/eot_labels.csv'
|
|||
|
|
label_hash = hashlib.sha256(labels_path.read_bytes()).hexdigest()
|
|||
|
|
soul = json.loads((RESULTS/'work-data-can-reply/summary.json').read_text())
|
|||
|
|
soul_latency = json.loads((RESULTS/'latency.json').read_text())
|
|||
|
|
maai_dir = ROOT/'experiments/maai-ch-kyoto/results'
|
|||
|
|
smart_dir = ROOT/'experiments/smart-turn-v3.2/results'
|
|||
|
|
maai = json.loads((maai_dir/'work-data-can-reply/summary.json').read_text())
|
|||
|
|
smart = json.loads((smart_dir/'work-data-can-reply/summary.json').read_text())
|
|||
|
|
assert soul['complete'] and soul['total_points']==44 and soul_latency['complete']
|
|||
|
|
assert soul['metadata']['label_sha256']==maai['config']['label_sha256']==smart['config']['label_sha256']==label_hash
|
|||
|
|
for call,checksum in soul['audio_sha256'].items():
|
|||
|
|
path=ROOT/'data/韩泰机器人抢话/音频'/f'{call}.mp3'
|
|||
|
|
assert hashlib.sha256(path.read_bytes()).hexdigest()==checksum==maai['config']['audio_sha256'][call]
|
|||
|
|
key=lambda r:(r['call_id'],r['decision_ms'])
|
|||
|
|
points=read(RESULTS/'work-data-can-reply/predictions.csv')
|
|||
|
|
smoke=read(RESULTS/'smoke/predictions.csv')[0]
|
|||
|
|
repeated=next(r for r in points if key(r)==key(smoke))
|
|||
|
|
repeat_check=dict(key=list(key(smoke)),
|
|||
|
|
probability_difference=abs(float(smoke['probability'])-float(repeated['probability'])),
|
|||
|
|
same_asr=smoke['asr_text']==repeated['asr_text'],
|
|||
|
|
same_raw_state=smoke['raw_state']==repeated['raw_state'])
|
|||
|
|
assert repeat_check['probability_difference'] < 1e-6 and repeat_check['same_asr'], repeat_check
|
|||
|
|
m={key(r):r for r in read(maai_dir/'work-data-can-reply/predictions.csv')}
|
|||
|
|
s={key(r):r for r in read(smart_dir/'work-data-can-reply/predictions.csv')}
|
|||
|
|
labels=read(labels_path)
|
|||
|
|
assert len({key(r) for r in points})==44
|
|||
|
|
assert set(map(key,points))==set(m)==set(s)==set(map(key,labels))
|
|||
|
|
rows=[]
|
|||
|
|
frames={}
|
|||
|
|
for f in read(RESULTS/'work-data-can-reply/frame-times.csv'):
|
|||
|
|
frames.setdefault(key(f),[]).append(f)
|
|||
|
|
for r in points:
|
|||
|
|
a,b=m[key(r)],s[key(r)]
|
|||
|
|
assert r['can_reply_now']==a['can_reply_now']==b['can_reply_now']
|
|||
|
|
history=frames[key(r)]
|
|||
|
|
assert len(history)==int(r['frame_count'])
|
|||
|
|
assert int(history[-1]['input_available_ms'])==int(r['decision_ms'])
|
|||
|
|
allow=0
|
|||
|
|
last_state='initial_hold'
|
|||
|
|
last_ms=r['audio_start_ms']
|
|||
|
|
for index,f in enumerate(history):
|
|||
|
|
assert int(f['frame'])==index and int(f['input_available_ms'])<=int(r['decision_ms'])
|
|||
|
|
if f['state']=='<|user_complete|>':
|
|||
|
|
allow=1;last_state=f['state'];last_ms=f['input_available_ms']
|
|||
|
|
elif f['state'] in ('<|user_incomplete|>','<|user_nonidle|>'):
|
|||
|
|
allow=0;last_state=f['state'];last_ms=f['input_available_ms']
|
|||
|
|
rows.append(dict(call_id=r['call_id'],decision_ms=r['decision_ms'],
|
|||
|
|
user_text_at_decision=r['user_text_at_decision'],can_reply_now=r['can_reply_now'],
|
|||
|
|
soulx_probability=float(r['probability']),soulx_05=int(float(r['probability'])>.5),
|
|||
|
|
soulx_08=int(float(r['probability'])>.8),soulx_raw_state=r['raw_state'],
|
|||
|
|
soulx_native_complete=int(r['raw_state']=='<|user_complete|>'),soulx_asr=r['asr_text'],
|
|||
|
|
soulx_corrected_complete=int(r['corrected_state']=='<|user_complete|>'),
|
|||
|
|
soulx_state_can_reply=allow,soulx_last_native_decision_state=last_state,
|
|||
|
|
soulx_last_native_decision_ms=last_ms,
|
|||
|
|
maai_probability=float(a['p_now_robot']),maai_05=int(float(a['p_now_robot'])>.5),
|
|||
|
|
maai_08=int(float(a['p_now_robot'])>.8),
|
|||
|
|
maai_turnbench_score=1-float(a['p_now_user']),
|
|||
|
|
maai_zero_false_hold=int(1-float(a['p_now_user'])>.11),
|
|||
|
|
smart_probability=float(b['probability']),smart_05=int(float(b['probability'])>.5),
|
|||
|
|
smart_08=int(float(b['probability'])>.8)))
|
|||
|
|
with (RESULTS/'comparison-points.csv').open('w',encoding='utf-8-sig',newline='') as f:
|
|||
|
|
w=csv.DictWriter(f,fieldnames=list(rows[0]));w.writeheader();w.writerows(rows)
|
|||
|
|
metric={k:metrics(rows,k) for k in ('soulx_05','soulx_08','soulx_native_complete','soulx_corrected_complete','soulx_state_can_reply',
|
|||
|
|
'maai_05','maai_08','maai_zero_false_hold','smart_05','smart_08')}
|
|||
|
|
assert metric['soulx_05']==soul['metrics']
|
|||
|
|
for x in ('tn','fp','fn','tp','accuracy'):
|
|||
|
|
assert metric['maai_05'][x]==maai['main_metrics'][x]
|
|||
|
|
assert metric['smart_05'][x]==smart['metrics'][x]
|
|||
|
|
baseline_latency=json.loads((ROOT/'experiments/latency-comparison.json').read_text())
|
|||
|
|
comparisons={base:dict(
|
|||
|
|
fixed=sum(r[base]!=int(r['can_reply_now']) and r['soulx_05']==int(r['can_reply_now']) for r in rows),
|
|||
|
|
regressed=sum(r[base]==int(r['can_reply_now']) and r['soulx_05']!=int(r['can_reply_now']) for r in rows))
|
|||
|
|
for base in ('maai_05','smart_05','smart_08')}
|
|||
|
|
native_comparisons={base:dict(
|
|||
|
|
fixed=sum(r[base]!=int(r['can_reply_now']) and r['soulx_state_can_reply']==int(r['can_reply_now']) for r in rows),
|
|||
|
|
regressed=sum(r[base]==int(r['can_reply_now']) and r['soulx_state_can_reply']!=int(r['can_reply_now']) for r in rows))
|
|||
|
|
for base in ('maai_05','smart_05','smart_08')}
|
|||
|
|
lowest_positive=min((r for r in rows if int(r['can_reply_now'])==1),
|
|||
|
|
key=lambda r:r['maai_turnbench_score'])
|
|||
|
|
zero_hold_scheme=dict(rule='1 - p_now[user] > 0.11',threshold=.11,
|
|||
|
|
metrics=metric['maai_zero_false_hold'],label_sha256=label_hash,
|
|||
|
|
selection='Supplementary operating point selected on these same 44 labels; not an independent validation or a guarantee of zero false holds on new audio.',
|
|||
|
|
lowest_positive=dict(call_id=lowest_positive['call_id'],
|
|||
|
|
decision_ms=lowest_positive['decision_ms'],
|
|||
|
|
score=lowest_positive['maai_turnbench_score'],
|
|||
|
|
text=lowest_positive['user_text_at_decision']),
|
|||
|
|
model_inference_unchanged=True,
|
|||
|
|
same_predictions_using_p_now_robot=all(
|
|||
|
|
r['maai_zero_false_hold']==int(r['maai_probability']>.11) for r in rows))
|
|||
|
|
assert zero_hold_scheme['metrics']['fn']==0
|
|||
|
|
assert zero_hold_scheme['same_predictions_using_p_now_robot']
|
|||
|
|
(maai_dir/'work-data-can-reply/zero-false-hold.json').write_text(
|
|||
|
|
json.dumps(zero_hold_scheme,ensure_ascii=False,indent=2)+'\n')
|
|||
|
|
summary=dict(label_sha256=label_hash,metrics=metric,changed_vs_baseline=comparisons,
|
|||
|
|
maai_zero_false_hold_scheme=zero_hold_scheme,
|
|||
|
|
repeated_point_check=repeat_check,
|
|||
|
|
native_can_reply_mapping='Supplementary post-hoc mapping: start hold; complete -> allow; incomplete/nonidle -> hold; idle/backchannel retain previous value. No timeout, no service resets, no external EOT signals, no future frames.',
|
|||
|
|
native_can_reply_changed_vs_baseline=native_comparisons,
|
|||
|
|
soulx_native_state_metrics=soul['native_state_metrics'],
|
|||
|
|
soulx_corrected_state_metrics=soul['corrected_state_metrics'],
|
|||
|
|
soulx_threshold_diagnostics=soul['thresholds'],
|
|||
|
|
latency=dict(soulx=soul_latency,local_baselines=baseline_latency),
|
|||
|
|
method=soul['method'],limitations=soul['limitations'])
|
|||
|
|
(RESULTS/'comparison-summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n')
|
|||
|
|
lines=['SoulX-Duplug / MaAI / Smart Turn：当前44个标注点对比',
|
|||
|
|
'', '数据：6段录音，can_reply=0/1各22点；标签和音频SHA256与本机基线一致。',
|
|||
|
|
'标签中29点来自xlsx草稿、15点没有明确审核信息，结果属于探索性标注点评估。',
|
|||
|
|
'', '准确率及误判（误放行：不该回复却允许；误阻止：可以回复却拦截）']
|
|||
|
|
for name,k in [('SoulX 条件完成分数 >0.5（预先定义的分数口径）','soulx_05'),('SoulX 条件完成分数 >0.8','soulx_08'),
|
|||
|
|
('SoulX 原生状态派生can_reply（辅助映射）','soulx_state_can_reply'),
|
|||
|
|
('MaAI p_now_robot >0.5','maai_05'),('MaAI p_now_robot >0.8','maai_08'),
|
|||
|
|
('MaAI 1-p_now_user >0.11（本批误阻止为0的补充方案）','maai_zero_false_hold'),
|
|||
|
|
('Smart Turn >0.5','smart_05'),('Smart Turn >0.8','smart_08')]:
|
|||
|
|
x=metric[k]
|
|||
|
|
lines.append(f"{name}：{x['accuracy']:.1%}；误放行{x['fp']}/22，误阻止{x['fn']}/22；TN/FP/FN/TP={x['tn']}/{x['fp']}/{x['fn']}/{x['tp']}")
|
|||
|
|
lines.extend(['',
|
|||
|
|
'MaAI补充方案：1-p_now_user >0.11；在本批44点中与p_now_robot >0.11的判断完全一致。',
|
|||
|
|
f"可回复点的最低分为{lowest_positive['maai_turnbench_score']:.9f}，来自{lowest_positive['call_id']} {lowest_positive['decision_ms']}ms：{lowest_positive['user_text_at_decision']}。",
|
|||
|
|
'阈值0.11在这批标签上选择，误阻止0/22、误放行17/22（77.27%），不能保证新数据仍零误阻止；模型推理及已测量计算延迟不变。',
|
|||
|
|
'在同为误阻止0的配置下，MaAI 0.11比Smart Turn 0.5少1个误放行，比Smart Turn 0.8多1个误放行。',
|
|||
|
|
'', 'SoulX原生五状态argmax恰为user_complete的诊断指标：'+json.dumps(metric['soulx_native_complete']),
|
|||
|
|
'SoulX内部状态修正后恰为user_complete的诊断指标：'+json.dumps(metric['soulx_corrected_complete']),
|
|||
|
|
'主指标在每个判断点比较complete/incomplete logits，排除idle/nonidle等类别后归一化；分数不是官方校准的can_reply概率。',
|
|||
|
|
'原生状态诊断与主指标不是同一种决策，不能把主指标宣称为完整官方服务效果。',
|
|||
|
|
'补充原生状态映射：开始为不可回复；complete置为可回复，incomplete/nonidle置为不可回复，idle/backchannel保持前值。只使用该标注点之前的原生预测，不加入超时、外部结束信号或音频清空。',
|
|||
|
|
'这是为区分瞬时完成事件与持续可回复状态而增加的辅助映射，保留预先定义的条件分数结果，未按标签调参。',
|
|||
|
|
'只在complete/incomplete间归一化，会忽略idle/nonidle等类别，可能产生虚高分数；条件分数结果不能直接代表原生SoulX性能。',
|
|||
|
|
'SoulX使用用户单声道及最多19.88秒真实音频；MaAI使用双声道及最多20秒；Smart Turn使用用户单声道及最多8秒。',
|
|||
|
|
'SoulX每个判断点重新初始化KV，直接使用原生infer，保留内部ASR和状态计算，不使用超时回复、完成后的服务重置或最终整句ASR。',
|
|||
|
|
'最终处理块在判断点前40ms结束，前向音频在判断点结束；不读标注点后的音频，也不输入标签转写。',
|
|||
|
|
'已知cb3238dc 13540和e304f740 11531存在标注/ASR对齐问题，未改动人工标签。',
|
|||
|
|
'', '计算延迟（跨机器配置对比）'])
|
|||
|
|
for name,x in [('MaAI / M4 CPU4线程，每100ms增量',baseline_latency['maai_4_threads_incremental']),
|
|||
|
|
('Smart Turn / M4 CPU4线程，8秒窗口',baseline_latency['smart_turn_4_threads']),
|
|||
|
|
('Smart Turn / M4 CPU1线程，8秒窗口',baseline_latency['smart_turn_1_thread']),
|
|||
|
|
('SoulX / RTX3090 + i5 CPU4线程，每160ms增量，含ASR',soul_latency['summary']['steady']['total_ms']),
|
|||
|
|
('SoulX / 同上，扣除cascade ASR调用时间',soul_latency['summary']['steady']['non_cascade_ms'])]:
|
|||
|
|
lines.append(f"{name}：均值{x['mean_ms']:.2f}ms，P95 {x['p95_ms']:.2f}ms，样本数{x['n']}")
|
|||
|
|
lines.extend([f"SoulX全段计算RTF={soul_latency['realtime_factor']:.3f}，超过160ms更新周期的帧比例={soul_latency['fraction_over_160ms']:.1%}。",
|
|||
|
|
'SoulX用CUDA同步后的墙钟计时，分解音频特征/分词器、ASR阶段和状态模型，模型加载与离线重放单独记录。',
|
|||
|
|
'扣除cascade ASR的数值仍包括ASR触发检测、文本处理等，不等同于纯状态网络时间。',
|
|||
|
|
'MaAI是100ms双声道增量，Smart Turn每次计算8秒窗口，SoulX是160ms增量；处理量和硬件不同，不能据此断言同机速度排名。',
|
|||
|
|
'Smart Turn包含约0.16ms重采样；MaAI和SoulX计时输入已为16kHz，重采样/文件解码不包含在增量计时内。',
|
|||
|
|
f"SoulX实际参数dtype={soul['metadata']['parameter_dtypes']}；官方配置写bf16但加载器没有转换，本次未添加自动混合精度。",
|
|||
|
|
f"SoulX点评估平均离线重放={sum(float(r['replay_inference_ms']) for r in points)/len(points):.2f}ms，不作为在线单步延迟。",
|
|||
|
|
f"SoulX热缓存加载耗时{soul['metadata']['load_ms']/1000:.2f}s（不包含Python导入），完整连续测试峰值GPU分配{soul_latency['peak_gpu_allocated_bytes']/2**30:.2f}GiB。",
|
|||
|
|
'160ms更新周期和40ms前向窗口没有计入计算耗时；约21.8%的帧计算时间超过160ms，应区分平均实时吞吐与局部排队延迟。',
|
|||
|
|
'原始最终实验日志有32次ASR list index out of range警告，官方适配器捕获后返回空文本；未过滤这些样本，日志保留以便进一步定位。',
|
|||
|
|
'', 'SoulX误判点（0.5）'])
|
|||
|
|
for r in rows:
|
|||
|
|
if r['soulx_05']!=int(r['can_reply_now']):
|
|||
|
|
lines.append(f"{r['call_id']} {r['decision_ms']}ms 标签={r['can_reply_now']} 预测={r['soulx_05']} p={r['soulx_probability']:.5f} 文本={r['user_text_at_decision']} ASR={r['soulx_asr']}")
|
|||
|
|
lines.extend(['','SoulX原生状态派生can_reply的误判点（辅助映射）'])
|
|||
|
|
for r in rows:
|
|||
|
|
if r['soulx_state_can_reply']!=int(r['can_reply_now']):
|
|||
|
|
lines.append(f"{r['call_id']} {r['decision_ms']}ms 标签={r['can_reply_now']} 预测={r['soulx_state_can_reply']} 最后决定状态={r['soulx_last_native_decision_state']} 文本={r['user_text_at_decision']} ASR={r['soulx_asr']}")
|
|||
|
|
lines.extend(['','本次数据的判断：',
|
|||
|
|
'MaAI在相同误阻止数量4点下，比SoulX原生状态辅助映射少7个误放行（7 vs 14），整体准确率最高。',
|
|||
|
|
'SoulX原生状态辅助映射相比Smart Turn 0.5少4个误放行，但多4个误阻止，准确率均为59.1%。',
|
|||
|
|
'SoulX误放行主要仍是完整短语中的停顿；原生状态辅助映射的4个误阻止集中在贵州省遵义市仁怀市、我姓宋、辽宁葫芦岛、姓赵，涉及地点/姓名和两个已知对齐疑点。',
|
|||
|
|
'条件两类分数整体偏高，0.8阈值仅拦截正确2/22个不可回复点，不适合直接当作当前业务的can_reply概率。',
|
|||
|
|
'在本次默认FP32、当前音频及映射配置下，SoulX未显示质量或计算延迟优势；不能推断其他数据、完整官方策略或BF16优化的表现。'])
|
|||
|
|
(RESULTS/'对比分析.txt').write_text('\n'.join(lines)+'\n')
|
|||
|
|
print(json.dumps(dict(metrics=metric,changed_vs_baseline=comparisons,
|
|||
|
|
soulx_latency=soul_latency['summary']['steady']['total_ms']),ensure_ascii=False,indent=2))
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__=='__main__':
|
|||
|
|
main()
|