171 lines
10 KiB
Python
171 lines
10 KiB
Python
"""Evaluate MaAI activity-projection proxies at exact annotated media times.
|
||
|
||
Each point uses a new model state and only preceding stereo audio, up to 20 s.
|
||
Outputs are mapped to can_reply without any end-signal or timeout gating.
|
||
"""
|
||
import argparse
|
||
from collections import Counter
|
||
import contextlib
|
||
import csv
|
||
import hashlib
|
||
import importlib.metadata
|
||
import io
|
||
import json
|
||
from pathlib import Path
|
||
import time
|
||
|
||
from smoke_run import HERE, ROOT, WEIGHTS, CPC, EXPECTED, Maai, MaaiInput, np, sf, soxr, torch
|
||
|
||
OUT = HERE / "results/work-data-can-reply"
|
||
LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
||
HISTORY = ROOT / "labels/EOT标注/工作数据/asr_history.csv"
|
||
RULES = ("p_now_robot", "p_future_robot", "both_robot")
|
||
|
||
|
||
def read(path):
|
||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||
return list(csv.DictReader(f))
|
||
|
||
|
||
def write(path, rows):
|
||
if not rows:
|
||
return
|
||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
|
||
|
||
def metrics(rows, score="p_now_robot", threshold=0.5):
|
||
counts = Counter((int(r["can_reply_now"]), int(float(r[score]) > threshold))
|
||
for r in rows if r["can_reply_now"] in ("0", "1"))
|
||
tn, fp, fn, tp = counts[0, 0], counts[0, 1], counts[1, 0], counts[1, 1]
|
||
total = tn+fp+fn+tp
|
||
ratio = lambda n, d: n/d if d else None
|
||
return dict(n=total, tn=tn, fp=fp, fn=fn, tp=tp,
|
||
accuracy=ratio(tn+tp, total), false_reply_rate=ratio(fp, fp+tn),
|
||
false_hold_rate=ratio(fn, fn+tp), recall=ratio(tp, tp+fn),
|
||
precision=ratio(tp, tp+fp), f1=ratio(2*tp, 2*tp+fp+fn))
|
||
|
||
|
||
def model():
|
||
# New instance also resets CPC recurrent state, not just VAP's KV cache.
|
||
with contextlib.redirect_stdout(io.StringIO()), torch.serialization.safe_globals([argparse.Namespace]):
|
||
return Maai(mode="vap", lang="ch_kyoto", audio_ch1=MaaiInput.Chunk(),
|
||
audio_ch2=MaaiInput.Chunk(), frame_rate=10, context_len_sec=20,
|
||
device="cpu", model_type="normal", mc=True, use_kv_cache=True,
|
||
cpc_model=str(CPC), local_model=str(WEIGHTS),
|
||
cache_dir=str(HERE / ".cache/huggingface"))
|
||
|
||
|
||
def main():
|
||
torch.set_num_threads(4)
|
||
torch.set_num_interop_threads(1)
|
||
label_hash = hashlib.sha256(LABELS.read_bytes()).hexdigest()
|
||
for path in (WEIGHTS, CPC):
|
||
assert hashlib.sha256(path.read_bytes()).hexdigest() == EXPECTED[path.name]
|
||
labels, history = read(LABELS), read(HISTORY)
|
||
keys = [(r["call_id"], r["decision_ms"]) for r in labels]
|
||
assert len(set(keys)) == len(keys)
|
||
previous_path = ROOT / "experiments/smart-turn-v3.2/results/work-data-can-reply/predictions.csv"
|
||
previous_summary = json.loads(previous_path.with_name("summary.json").read_text())
|
||
if previous_summary["config"]["label_sha256"] != label_hash:
|
||
raise ValueError("Smart Turn baseline labels differ from current labels; rerun that baseline first")
|
||
previous = {(r["call_id"], r["decision_ms"]): r for r in read(previous_path)}
|
||
audio_cache, rows, audio_hashes = {}, [], {}
|
||
began = time.perf_counter()
|
||
for index, label in enumerate(labels):
|
||
call, decision = label["call_id"], int(label["decision_ms"])
|
||
if call not in audio_cache:
|
||
path = ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3"
|
||
audio, sr = sf.read(path, dtype="float32", always_2d=True)
|
||
assert audio.shape[1] == 2
|
||
audio_cache[call] = audio, sr
|
||
audio_hashes[call] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||
audio, sr = audio_cache[call]
|
||
# End aligned at decision_ms, so every point's final 100ms frame ends
|
||
# exactly there. At file beginning drop <100ms from the leading edge.
|
||
duration_ms = min(20000, decision // 100 * 100)
|
||
if duration_ms == 0:
|
||
raise ValueError(f"Insufficient history at {call} {decision}")
|
||
origin = decision-duration_ms
|
||
start, end = origin*sr//1000, decision*sr//1000
|
||
assert 0 <= start < end <= len(audio)
|
||
# Crop BEFORE resampling: resampler cannot observe future audio.
|
||
segment = np.ascontiguousarray(soxr.resample(audio[start:end], sr, 16000, quality="HQ"), dtype=np.float32)
|
||
count = duration_ms//100
|
||
assert len(segment) == count*1600
|
||
maai = model()
|
||
maai.CALC_PROCESS_TIME_INTERVAL = 10**9
|
||
inference_began = time.perf_counter()
|
||
for frame in range(count):
|
||
position = frame*1600
|
||
maai.process(np.ascontiguousarray(segment[position:position+1600, 0]),
|
||
np.ascontiguousarray(segment[position:position+1600, 1]))
|
||
result = maai.result_dict_queue.get_nowait()
|
||
assert maai.result_dict_queue.empty()
|
||
inference_ms = (time.perf_counter()-inference_began)*1000
|
||
values = {}
|
||
for key in ("p_now", "p_future", "vad"):
|
||
values[key] = np.asarray(result[key], dtype=float).reshape(-1)
|
||
assert len(values[key]) == 2
|
||
assert np.isfinite(values[key]).all() and ((values[key] >= 0) & (values[key] <= 1)).all()
|
||
now, future = float(values["p_now"][1]), float(values["p_future"][1])
|
||
expected = int(label["can_reply_now"])
|
||
baseline = previous[(call, label["decision_ms"])]
|
||
call_history = [r for r in history if r["call_id"] == call]
|
||
agent_active = any(r["role"] == "agent" and int(r["start_ms"]) < decision < int(r["end_ms"]) for r in call_history)
|
||
user_active = any(r["role"] == "user" and int(r["start_ms"]) < decision < int(r["end_ms"]) for r in call_history)
|
||
r = dict(label)
|
||
r.update(p_now_user=float(values["p_now"][0]), p_now_robot=now,
|
||
p_future_user=float(values["p_future"][0]), p_future_robot=future,
|
||
both_robot=min(now, future), vad_user=float(values["vad"][0]), vad_robot=float(values["vad"][1]),
|
||
prediction_05=int(now > 0.5), correct_05=int(int(now > 0.5) == expected),
|
||
prediction_08=int(now > 0.8), correct_08=int(int(now > 0.8) == expected),
|
||
future_prediction_05=int(future > 0.5), both_prediction_05=int(min(now, future) > 0.5),
|
||
smart_turn_probability=float(baseline["probability"]),
|
||
smart_turn_prediction_05=int(baseline["prediction"]),
|
||
smart_turn_prediction_08=int(float(baseline["probability"]) > 0.8),
|
||
audio_start_ms=origin, audio_end_ms=decision, context_ms=duration_ms,
|
||
frame_count=count, replay_inference_ms=inference_ms,
|
||
asr_agent_active_at_point=int(agent_active), asr_user_segment_crosses_point=int(user_active),
|
||
label_status="draft" if label["annotator"] == "xlsx_draft" else "unconfirmed_manual")
|
||
rows.append(r)
|
||
print(f"{index+1}/{len(labels)} {call[:8]} {decision}: label={expected}, p_now_robot={now:.4f}, p_future_robot={future:.4f}", flush=True)
|
||
del maai
|
||
OUT.mkdir(parents=True, exist_ok=True)
|
||
write(OUT / "predictions.csv", rows)
|
||
write(OUT / "errors.csv", [r for r in rows if not r["correct_05"]])
|
||
thresholds = [dict(rule=rule, threshold=threshold, **metrics(rows, rule, threshold))
|
||
for rule in RULES for threshold in (0.5, 0.7, 0.8, 0.9, 0.95, 0.98)]
|
||
write(OUT / "thresholds.csv", thresholds)
|
||
groups = {field: {value: metrics([r for r in rows if r[field] == value])
|
||
for value in sorted({r[field] for r in rows})}
|
||
for field in ("call_id", "reason", "label_status", "asr_agent_active_at_point")}
|
||
summary = dict(total_points=len(rows), main_rule="p_now_robot > 0.5", main_metrics=metrics(rows),
|
||
rule_metrics={rule: metrics(rows, rule) for rule in RULES},
|
||
thresholds=thresholds, grouped=groups,
|
||
smart_turn_comparison={str(t): metrics(rows, "smart_turn_probability", t) for t in (0.5, 0.8)},
|
||
changed_vs_smart_turn=dict(
|
||
fixed=sum(int(r["smart_turn_prediction_05"]) != int(r["can_reply_now"]) and r["correct_05"] for r in rows),
|
||
regressed=sum(int(r["smart_turn_prediction_05"]) == int(r["can_reply_now"]) and not r["correct_05"] for r in rows)),
|
||
config=dict(maai_version=importlib.metadata.version("maai"), torch_version=torch.__version__,
|
||
device="cpu", threads=4, frame_rate=10, context_len_sec=20, use_kv_cache=True,
|
||
weight_variant="vap_mc", weight_sha256=EXPECTED, label_sha256=label_hash,
|
||
history_sha256=hashlib.sha256(HISTORY.read_bytes()).hexdigest(), audio_sha256=audio_hashes,
|
||
script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()),
|
||
elapsed_seconds=time.perf_counter()-began,
|
||
method="At each exact decision_ms a fresh MaAI/CPC state replays a preceding <=20s stereo window in 100ms frames. Crop before resampling. No future input, no labels as model input, no EOT or VAD gates, no forced timeout replies. Classification is a stated proxy rule, not an official calibrated can_reply output.",
|
||
limitations=["29 xlsx_draft and 15 rows without explicit verification metadata; exploratory selected-point scores.",
|
||
"The proxy p_now[robot] forecasts relative speaker activity over 0–0.6s; p_future forecasts 0.6–2s. Neither directly predicts task completion.",
|
||
"Independent 20s window restart differs from continuous whole-call state, particularly CPC recurrent state and deeply contextualized KV history.",
|
||
"MaAI uses both recorded channels and up to20s; Smart Turn baseline uses user channel and up to8s. Comparison evaluates these input configurations, not identical-input model capability.",
|
||
"Robot speech already present in recording can influence activity forecasts; agent-active checkpoints are separately marked.",
|
||
"ASR alignment issues previously found at cb3238dc 13540 and e304f740 11531 remain; labels unchanged."])
|
||
(OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2)+"\n")
|
||
print(json.dumps({k:summary[k] for k in ("main_metrics", "rule_metrics", "changed_vs_smart_turn", "elapsed_seconds")}, ensure_ascii=False, indent=2))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|