170 lines
10 KiB
Python
170 lines
10 KiB
Python
|
"""Score VAP annotation points from already available continuous stream outputs.
|
||
|
|
|
||
|
|
No model inference, state resets, interpolation, or changes to archived results.
|
||
|
|
"""
|
||
|
|
import argparse
|
||
|
|
from bisect import bisect_right
|
||
|
|
from collections import Counter
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
import csv
|
||
|
|
import hashlib
|
||
|
|
import json
|
||
|
|
import math
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
HERE = Path(__file__).resolve().parent
|
||
|
|
ROOT = HERE.parents[1]
|
||
|
|
LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
||
|
|
SOURCES = {
|
||
|
|
"maai": ("score-timeline.json", "cpc-stereo", "CPC 降噪双声道"),
|
||
|
|
"maai_user_only": ("user-only/score-timeline.json", "cpc-user-only", "CPC 降噪机器人静音"),
|
||
|
|
"maai_mimi_standard": ("mimi-standard/score-timeline.json", "mimi-standard", "Mimi 标准双声道"),
|
||
|
|
"maai_mimi_mc": ("mimi-mc/score-timeline.json", "mimi-mc", "Mimi 降噪双声道"),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def sha(path):
|
||
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||
|
|
|
||
|
|
|
||
|
|
def last_available(points, decision_ms, max_age_ms):
|
||
|
|
"""The same inclusive past-only, bounded-age rule used by Studio scoreAt."""
|
||
|
|
index = bisect_right(points, decision_ms, key=lambda p: p[0]) - 1
|
||
|
|
if index < 0 or decision_ms - points[index][0] > max_age_ms:
|
||
|
|
return None
|
||
|
|
return points[index]
|
||
|
|
|
||
|
|
|
||
|
|
def classification(rows, threshold=.5):
|
||
|
|
labeled = [r for r in rows if str(r['can_reply_now']) in ('0', '1')]
|
||
|
|
available = [r for r in labeled if r['status'] == 'available']
|
||
|
|
counts = Counter((int(r['can_reply_now']), int(float(r['score']) > threshold)) for r in available)
|
||
|
|
tn, fp, fn, tp = counts[0,0], counts[0,1], counts[1,0], counts[1,1]
|
||
|
|
n = tn + fp + fn + tp
|
||
|
|
ratio = lambda a,b: a/b if b else None
|
||
|
|
return dict(n=n, requested_labeled=len(labeled), unavailable_labeled=len(labeled)-n,
|
||
|
|
tn=tn, fp=fp, fn=fn, tp=tp, correct=tn+tp, accuracy=ratio(tn+tp,n),
|
||
|
|
false_reply_rate=ratio(fp,fp+tn), false_hold_rate=ratio(fn,fn+tp),
|
||
|
|
recall=ratio(tp,tp+fn), precision=ratio(tp,tp+fp), f1=ratio(2*tp,2*tp+fp+fn))
|
||
|
|
|
||
|
|
|
||
|
|
def write_csv(path, rows):
|
||
|
|
if not rows:
|
||
|
|
return
|
||
|
|
with path.open('w', encoding='utf-8-sig', newline='') as f:
|
||
|
|
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||
|
|
writer.writeheader()
|
||
|
|
writer.writerows(rows)
|
||
|
|
|
||
|
|
|
||
|
|
def refresh_streaming(model):
|
||
|
|
source_file, directory, name = SOURCES[model]
|
||
|
|
source_path = HERE / 'results' / source_file
|
||
|
|
source_hash, label_hash = sha(source_path), sha(LABELS)
|
||
|
|
source = json.loads(source_path.read_text())
|
||
|
|
if source['model'] != model or source['schema_version'] != 1:
|
||
|
|
raise ValueError('Unexpected source export')
|
||
|
|
interval = source['sample_interval_ms']
|
||
|
|
assert math.isfinite(interval) and interval > 0
|
||
|
|
with LABELS.open(encoding='utf-8-sig', newline='') as f:
|
||
|
|
labels = list(csv.DictReader(f))
|
||
|
|
assert len({(r['call_id'], int(r['decision_ms'])) for r in labels}) == len(labels)
|
||
|
|
calls = {c['call_id']: c for c in source['calls']}
|
||
|
|
assert len(calls) == len(source['calls'])
|
||
|
|
assert {r['call_id'] for r in labels} <= calls.keys(), 'Missing annotated calls'
|
||
|
|
rows, exported_calls = [], []
|
||
|
|
for call_id, call in calls.items():
|
||
|
|
audio = ROOT / 'data' / call['audio_path']
|
||
|
|
assert audio.resolve().is_relative_to((ROOT/'data').resolve()), 'Audio outside project data'
|
||
|
|
assert sha(audio) == call['audio_sha256'], 'Source recording changed; rerun inference'
|
||
|
|
points = call['points']
|
||
|
|
assert all(len(p)==2 and math.isfinite(p[0]) and math.isfinite(p[1]) and
|
||
|
|
0 <= p[0] <= call['duration_ms'] and 0 <= p[1] <= 1 for p in points)
|
||
|
|
assert all(a[0] < b[0] for a,b in zip(points,points[1:]))
|
||
|
|
checkpoints, sources = [], []
|
||
|
|
for label in sorted([r for r in labels if r['call_id']==call_id], key=lambda r:int(r['decision_ms'])):
|
||
|
|
decision = int(label['decision_ms'])
|
||
|
|
assert 0 <= decision <= call['duration_ms'], 'Annotation outside recording'
|
||
|
|
point = last_available(points, decision, interval*2)
|
||
|
|
row = dict(label)
|
||
|
|
row.update(status='available' if point is not None else 'unavailable',
|
||
|
|
score_time_ms=point[0] if point is not None else '',
|
||
|
|
score_age_ms=decision-point[0] if point is not None else '',
|
||
|
|
score=point[1] if point is not None else '',
|
||
|
|
prediction_05=int(point[1]>.5) if point is not None else '')
|
||
|
|
rows.append(row)
|
||
|
|
if point is not None:
|
||
|
|
checkpoints.append([decision, point[1]])
|
||
|
|
sources.append([decision, point[0]])
|
||
|
|
exported_calls.append(dict(call_id=call_id, audio_path=call['audio_path'], audio_sha256=call['audio_sha256'],
|
||
|
|
duration_ms=call['duration_ms'], points=points,
|
||
|
|
checkpoints=checkpoints, checkpoint_sources=sources))
|
||
|
|
provenance = dict(source_export=str(source_path.relative_to(ROOT)), source_export_sha256=source_hash,
|
||
|
|
source_inference_provenance=source.get('provenance', {}),
|
||
|
|
source_model_sha256=source.get('model_sha256', {}),
|
||
|
|
label_sha256=label_hash, evaluation_script_sha256=sha(Path(__file__)))
|
||
|
|
method = ('Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with '
|
||
|
|
'availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. '
|
||
|
|
'No state reset, future score, interpolation or independent-window checkpoint. '
|
||
|
|
'Availability includes source resampler buffering; score is the stored activity-projection proxy.')
|
||
|
|
exported = dict(schema_version=1, model=model, name=source['name'], score=source['score'],
|
||
|
|
checkpoint_mode='streaming', sample_interval_ms=interval, method=method,
|
||
|
|
max_score_age_ms=interval*2, provenance=provenance, calls=exported_calls)
|
||
|
|
summary = dict(model=model, name=name, evaluation_mode='streaming',
|
||
|
|
created_at=datetime.now(timezone.utc).isoformat(), threshold=.5,
|
||
|
|
rule='1 - p_now[user] > 0.5', main_metrics=classification(rows),
|
||
|
|
thresholds=[dict(threshold=t, **classification(rows,t)) for t in (.11,.5,.7,.8,.9,.95,.98)],
|
||
|
|
per_call={c:classification([r for r in rows if r['call_id']==c]) for c in calls},
|
||
|
|
method=method, provenance=provenance,
|
||
|
|
limitations=['Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.',
|
||
|
|
'Uses stored rounded continuous scores; no new inference or model-speed measurement.',
|
||
|
|
'No prior output or stale output is unavailable, not a negative/zero prediction.',
|
||
|
|
'Archived independent-window results are a separate evaluation protocol.'])
|
||
|
|
assert sha(source_path)==source_hash and sha(LABELS)==label_hash, 'Inputs changed during evaluation'
|
||
|
|
output = HERE/'results/streaming'/directory
|
||
|
|
output.mkdir(parents=True, exist_ok=True)
|
||
|
|
write_csv(output/'predictions.csv', rows)
|
||
|
|
(output/'summary.json').write_text(json.dumps(summary,ensure_ascii=False,indent=2)+'\n')
|
||
|
|
(output/'score-timeline.json').write_text(json.dumps(exported,ensure_ascii=False,separators=(',',':'))+'\n')
|
||
|
|
return summary, rows
|
||
|
|
|
||
|
|
|
||
|
|
def main(selected):
|
||
|
|
summaries, rows_by_model = {}, {}
|
||
|
|
for model in SOURCES if selected=='all' else (selected,):
|
||
|
|
summary, rows = refresh_streaming(model)
|
||
|
|
summaries[model], rows_by_model[model] = summary, rows
|
||
|
|
print(summary['name'], json.dumps(summary['main_metrics'],ensure_ascii=False),flush=True)
|
||
|
|
if selected != 'all':
|
||
|
|
return
|
||
|
|
label_hashes = {s['provenance']['label_sha256'] for s in summaries.values()}
|
||
|
|
assert len(label_hashes)==1, 'Models use different labels'
|
||
|
|
base = summaries['maai']['main_metrics']['accuracy']
|
||
|
|
for summary in summaries.values():
|
||
|
|
accuracy = summary['main_metrics']['accuracy']
|
||
|
|
summary['accuracy_delta_pp_vs_cpc_stereo'] = (accuracy-base)*100 if accuracy is not None and base is not None else None
|
||
|
|
summary['relative_accuracy_change_vs_cpc_stereo'] = (accuracy-base)/base if accuracy is not None and base else None
|
||
|
|
output = HERE/'results/streaming'
|
||
|
|
result = dict(evaluation_mode='streaming', label_sha256=next(iter(label_hashes)), models=summaries)
|
||
|
|
(output/'comparison-summary.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
|
||
|
|
lines = ['VAP 流式场景判断点对比（阈值严格 > 0.5）', '',
|
||
|
|
'模型 | 有效点/有标签点 | 正确数 | 准确率 | 误放行 FP | 误阻止 FN | 对 CPC 双声道变化']
|
||
|
|
for model, summary in summaries.items():
|
||
|
|
m = summary['main_metrics']
|
||
|
|
accuracy = f"{m['accuracy']:.2%}" if m['accuracy'] is not None else '不可用'
|
||
|
|
delta = summary['accuracy_delta_pp_vs_cpc_stereo']
|
||
|
|
delta_text = f'{delta:+.2f} 个百分点' if delta is not None else '不可用'
|
||
|
|
lines.append(f"{summary['name']} | {m['n']}/{m['requested_labeled']} | {m['correct']} | {accuracy} | {m['fp']} | {m['fn']} | {delta_text}")
|
||
|
|
lines += ['', '1. 本表使用连续推理在判断时刻已经产生的最新分数，与前端 VAP 圆点和悬停分数一致；不使用独立窗口结果。',
|
||
|
|
'2. 每个有效点记录 decision_ms、score_time_ms、score_age_ms；启动期或超过两倍更新间隔的旧分数不计为 0，而是不可用。',
|
||
|
|
'3. 数据为六通录音中的选定点，只做描述性比较；下一步宜扩充按通话隔离的测试集，再在验证集上选择阈值。',
|
||
|
|
'', '复现：在 MaAI 独立 uv 环境运行 python evaluate_streaming.py all。原独立窗口结果保留在原目录，另作历史诊断。']
|
||
|
|
(output/'对比分析.txt').write_text('\n'.join(lines)+'\n')
|
||
|
|
print('\n'.join(lines))
|
||
|
|
|
||
|
|
|
||
|
|
if __name__=='__main__':
|
||
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
||
|
|
parser.add_argument('model',choices=('all',*SOURCES),default='all',nargs='?')
|
||
|
|
main(parser.parse_args().model)
|