248 lines
14 KiB
Python
248 lines
14 KiB
Python
"""Evaluate VAP with channel 0=user and channel 1=zeros; export Studio scores.
|
|
|
|
Reuse the locked MaAI environment/weights. Independent point replays and the
|
|
continuous timeline are separate experiments. Never use annotation text as input.
|
|
"""
|
|
import argparse
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import importlib.metadata as metadata
|
|
import json
|
|
import platform
|
|
import random
|
|
import sys
|
|
import time
|
|
|
|
from smoke_run import HERE, ROOT, EXPECTED, WEIGHTS, CPC, np, sf, soxr, torch
|
|
from evaluate_can_reply import model, metrics, read, write
|
|
|
|
OUT = HERE / "results/user-only"
|
|
LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
|
HISTORY = LABELS.with_name("asr_history.csv")
|
|
SEED = 42
|
|
SCORE = "can_reply_score"
|
|
|
|
|
|
def sha(path):
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def seed():
|
|
random.seed(SEED)
|
|
np.random.seed(SEED)
|
|
torch.manual_seed(SEED)
|
|
|
|
|
|
def channels(pcm, robot_silent):
|
|
if pcm.ndim != 2 or pcm.shape[1] != 2:
|
|
raise ValueError("Expected channel 0=user, channel 1=robot")
|
|
if robot_silent:
|
|
# Do not mix, duplicate user audio, remove pauses, or alter source PCM.
|
|
result = np.zeros(pcm.shape, dtype=np.float32)
|
|
result[:, 0] = pcm[:, 0]
|
|
return result
|
|
return np.ascontiguousarray(pcm, dtype=np.float32)
|
|
|
|
|
|
def point_audio(pcm, sample_rate, decision_ms, robot_silent):
|
|
duration = min(20000, decision_ms // 100 * 100)
|
|
if duration <= 0:
|
|
raise ValueError("Point has less than 100 ms of history")
|
|
origin = decision_ms - duration
|
|
start, end = origin * sample_rate // 1000, decision_ms * sample_rate // 1000
|
|
if not 0 <= start < end <= len(pcm):
|
|
raise ValueError("Point outside recording")
|
|
# Crop received history BEFORE resampling, then mute before any model step.
|
|
segment = channels(pcm[start:end], robot_silent)
|
|
segment = np.ascontiguousarray(soxr.resample(segment, sample_rate, 16000, quality="HQ"), dtype=np.float32)
|
|
assert len(segment) == duration // 100 * 1600
|
|
if robot_silent:
|
|
assert np.count_nonzero(segment[:, 1]) == 0
|
|
return segment, origin
|
|
|
|
|
|
def replay(segment):
|
|
seed()
|
|
maai = model() # Reset both the CPC recurrent state and VAP KV cache.
|
|
maai.CALC_PROCESS_TIME_INTERVAL = 10**9
|
|
began = time.perf_counter()
|
|
for start in range(0, len(segment), 1600):
|
|
frame = segment[start:start + 1600]
|
|
assert len(frame) == 1600
|
|
maai.process(np.ascontiguousarray(frame[:, 0]), np.ascontiguousarray(frame[:, 1]))
|
|
result = maai.result_dict_queue.get_nowait()
|
|
assert maai.result_dict_queue.empty()
|
|
elapsed = (time.perf_counter() - began) * 1000
|
|
values = {key: np.asarray(result[key], dtype=float).reshape(-1) for key in ("p_now", "p_future", "vad")}
|
|
for value in values.values():
|
|
assert value.shape == (2,) and np.isfinite(value).all()
|
|
assert ((value >= 0) & (value <= 1)).all()
|
|
return values, elapsed
|
|
|
|
|
|
def score(values):
|
|
# Match the existing Studio score exactly. Speaker outputs have a small
|
|
# normalization epsilon, so 1-p_now[user] is not substituted by p_now[robot].
|
|
return 1.0 - float(values["p_now"][0])
|
|
|
|
|
|
def witness(pcm, sr, decision):
|
|
segment, _ = point_audio(pcm, sr, decision, True)
|
|
original_user = pcm[:, 0].copy()
|
|
changed = pcm.copy()
|
|
rng = np.random.default_rng(SEED)
|
|
changed[:, 1] = rng.normal(0, .1, len(changed))
|
|
changed[decision * sr // 1000:, 0] = rng.normal(0, .1, len(changed) - decision * sr // 1000)
|
|
other, _ = point_audio(changed, sr, decision, True)
|
|
assert np.array_equal(segment, other), "Robot/future audio leaked into model input"
|
|
assert np.array_equal(pcm[:, 0], original_user), "Source user PCM was mutated"
|
|
a, _ = replay(segment)
|
|
b, _ = replay(other)
|
|
assert all(np.array_equal(a[k], b[k]) for k in a), "Seeded inference is not repeatable"
|
|
print(f"WITNESS CPU seed={SEED}: robot/future invariance and repeat passed; score={score(a):.7f}", flush=True)
|
|
return dict(seed=SEED, robot_and_future_invariance=True, repeat_identical=True, score=score(a))
|
|
|
|
|
|
def timeline(pcm, sr):
|
|
seed()
|
|
maai = model()
|
|
maai.CALC_PROCESS_TIME_INTERVAL = 10**9
|
|
resampler = soxr.ResampleStream(sr, 16000, 2, dtype="float32", quality="HQ")
|
|
pending = np.empty((0, 2), dtype=np.float32)
|
|
points = []
|
|
for start in range(0, len(pcm), sr // 10):
|
|
end = min(len(pcm), start + sr // 10)
|
|
block = resampler.resample_chunk(channels(pcm[start:end], True), last=end == len(pcm))
|
|
pending = np.concatenate([pending, block])
|
|
latest = None
|
|
while len(pending) >= 1600:
|
|
frame, pending = pending[:1600], pending[1600:]
|
|
assert np.count_nonzero(frame[:, 1]) == 0
|
|
maai.process(np.ascontiguousarray(frame[:, 0]), np.ascontiguousarray(frame[:, 1]))
|
|
result = maai.result_dict_queue.get_nowait()
|
|
assert maai.result_dict_queue.empty()
|
|
latest = 1.0 - float(result["p_now"][0])
|
|
assert np.isfinite(latest) and 0 <= latest <= 1
|
|
if latest is not None:
|
|
# Availability time includes streaming resampler buffering. No
|
|
# unreceived audio or trailing synthetic silence is sent to VAP.
|
|
points.append([round(end * 1000 / sr, 3), round(latest, 7)])
|
|
return points
|
|
|
|
|
|
def provenance():
|
|
return dict(machine=platform.platform(), hardware="Apple M4 Mac mini / 16 GB", python=sys.version,
|
|
environment="experiments/maai-ch-kyoto/.venv (uv.lock)", device="cpu", threads=4,
|
|
dependencies={p: metadata.version(p) for p in ("maai", "torch", "transformers", "numpy", "soundfile", "soxr")},
|
|
sample_rate=16000, frame_ms=100, context_ms=20000, use_kv_cache=True, seed=SEED,
|
|
channel_mapping={"0": "original user", "1": "all-zero robot input"},
|
|
weight_sha256=EXPECTED, label_sha256=sha(LABELS), history_sha256=sha(HISTORY),
|
|
uv_lock_sha256=sha(HERE / "uv.lock"), script_sha256=sha(HERE / "run_user_only.py"),
|
|
model_factory_sha256=sha(HERE / "evaluate_can_reply.py"), loader_sha256=sha(HERE / "smoke_run.py"))
|
|
|
|
|
|
def main(mode):
|
|
torch.set_num_threads(4)
|
|
torch.set_num_interop_threads(1)
|
|
for path in (WEIGHTS, CPC):
|
|
if sha(path) != EXPECTED[path.name]:
|
|
raise ValueError(f"Weight checksum mismatch: {path}")
|
|
labels = read(LABELS)
|
|
keys = [(r["call_id"], int(r["decision_ms"])) for r in labels]
|
|
assert len(set(keys)) == len(keys), "Duplicate annotation key"
|
|
annotated = [r for r in labels if r["can_reply_now"] in ("0", "1")]
|
|
if not annotated:
|
|
raise ValueError("No binary annotation points")
|
|
prov = provenance()
|
|
audio_cache, audio_hashes = {}, {}
|
|
for call in sorted({r["call_id"] for r in labels}):
|
|
path = ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3"
|
|
pcm, sr = sf.read(path, dtype="float32", always_2d=True)
|
|
assert pcm.shape[1] == 2
|
|
audio_cache[call] = pcm, sr
|
|
audio_hashes[call] = sha(path)
|
|
first = annotated[0]
|
|
began = time.perf_counter()
|
|
checks = witness(*audio_cache[first["call_id"]], int(first["decision_ms"]))
|
|
if mode == "smoke":
|
|
print(json.dumps(dict(witness=checks, provenance=prov), ensure_ascii=False, indent=2))
|
|
return
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
rows = []
|
|
for index, label in enumerate(annotated):
|
|
call, decision = label["call_id"], int(label["decision_ms"])
|
|
pcm, sr = audio_cache[call]
|
|
silent_input, origin = point_audio(pcm, sr, decision, True)
|
|
silent, elapsed = replay(silent_input)
|
|
stereo_input, stereo_origin = point_audio(pcm, sr, decision, False)
|
|
assert origin == stereo_origin and np.array_equal(silent_input[:, 0], stereo_input[:, 0])
|
|
stereo, stereo_elapsed = replay(stereo_input)
|
|
value, baseline = score(silent), score(stereo)
|
|
row = dict(label)
|
|
row.update(can_reply_score=value, stereo_can_reply_score=baseline,
|
|
p_now_user=float(silent["p_now"][0]), p_now_robot=float(silent["p_now"][1]),
|
|
p_future_user=float(silent["p_future"][0]), p_future_robot=float(silent["p_future"][1]),
|
|
vad_user=float(silent["vad"][0]), vad_robot=float(silent["vad"][1]),
|
|
prediction_05=int(value > .5), correct_05=int(int(value > .5) == int(label["can_reply_now"])),
|
|
stereo_prediction_05=int(baseline > .5),
|
|
audio_start_ms=origin, audio_end_ms=decision, context_ms=decision-origin,
|
|
robot_input_peak_abs=float(np.max(np.abs(silent_input[:, 1]))),
|
|
replay_inference_ms=elapsed, stereo_replay_inference_ms=stereo_elapsed)
|
|
rows.append(row)
|
|
print(f"{index+1}/{len(annotated)} {call[:8]} {decision}: label={label['can_reply_now']}, user_only={value:.4f}, stereo={baseline:.4f}", flush=True)
|
|
# Guard against labels/history being edited during the run.
|
|
assert prov["label_sha256"] == sha(LABELS) and prov["history_sha256"] == sha(HISTORY)
|
|
write(OUT / "predictions.csv", rows)
|
|
# write() intentionally skips empty lists; remove any stale errors instead.
|
|
errors = [r for r in rows if not r["correct_05"]]
|
|
if errors:
|
|
write(OUT / "errors.csv", errors)
|
|
else:
|
|
(OUT / "errors.csv").unlink(missing_ok=True)
|
|
thresholds = [dict(threshold=t, **metrics(rows, SCORE, t)) for t in (.11, .5, .7, .8, .9, .95, .98)]
|
|
write(OUT / "thresholds.csv", thresholds)
|
|
summary = dict(model="VAP / MaAI (user only, silent robot)", total_points=len(rows),
|
|
created_at=datetime.now(timezone.utc).isoformat(), main_rule="1 - p_now[user] > 0.5",
|
|
main_metrics=metrics(rows, SCORE), thresholds=thresholds,
|
|
same_point_stereo_reference=metrics(rows, "stereo_can_reply_score"),
|
|
changed_vs_stereo=dict(fixed=sum(r["stereo_prediction_05"] != int(r["can_reply_now"]) and bool(r["correct_05"]) for r in rows),
|
|
regressed=sum(r["stereo_prediction_05"] == int(r["can_reply_now"]) and not r["correct_05"] for r in rows)),
|
|
per_call={call: metrics([r for r in rows if r["call_id"] == call], SCORE) for call in audio_cache},
|
|
annotation_sources=dict(Counter(r["annotator"] or "unspecified" for r in rows)),
|
|
skipped_unlabeled=len(labels)-len(annotated), witness=checks,
|
|
provenance={**prov, "audio_sha256": audio_hashes},
|
|
method="At each exact point, independently reset CPC/VAP and replay <=20s in 100ms frames. User channel unchanged; robot channel zero for the full context before resampling. Also rerun the stereo control on identical current points. No annotation transcripts, future audio, end signal, silence gate, or timeout. Score is an activity-projection proxy, not calibrated EOT probability.",
|
|
limitations=["Exploratory selected-point evaluation, not a held-out benchmark or endpointing-policy evaluation.",
|
|
"Continuous full-call KV/CPC state differs from independent point replay; curve and checkpoint scores may differ.",
|
|
"Zeroing the robot channel cannot remove robot leakage already present in the user channel.",
|
|
"Stereo reference here is rerun at current annotation points; existing legacy frontend checkpoints are preserved separately."])
|
|
if mode == "all":
|
|
calls = []
|
|
for call, (pcm, sr) in audio_cache.items():
|
|
points = timeline(pcm, sr)
|
|
call_rows = sorted([r for r in rows if r["call_id"] == call], key=lambda r: int(r["decision_ms"]))
|
|
calls.append(dict(call_id=call, audio_path=f"韩泰机器人抢话/音频/{call}.mp3", audio_sha256=audio_hashes[call],
|
|
duration_ms=round(len(pcm)*1000/sr, 3), points=points,
|
|
checkpoints=[[int(r["decision_ms"]), round(r[SCORE], 7)] for r in call_rows]))
|
|
print(f"{call[:8]} timeline: {len(points)} scores, {len(call_rows)} checkpoints", flush=True)
|
|
assert prov["label_sha256"] == sha(LABELS) and prov["history_sha256"] == sha(HISTORY)
|
|
exported = dict(schema_version=1, model="maai_user_only", name="VAP / MaAI（仅用户）", score="1 - p_now[user]",
|
|
sample_interval_ms=100, method="Continuous user + silent-robot stereo input, 16kHz, 10Hz, 20s KV cache. Robot channel zeroed before causal streaming resampling. Timestamps include resampling buffering. Checkpoints are independent <=20s replays.",
|
|
provenance=summary["provenance"], calls=calls)
|
|
(OUT / "score-timeline.json").write_text(json.dumps(exported, ensure_ascii=False, separators=(",", ":")) + "\n")
|
|
summary["timeline_points"] = sum(len(c["points"]) for c in calls)
|
|
summary["elapsed_seconds"] = time.perf_counter()-began
|
|
(OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
|
|
print(json.dumps({k: summary[k] for k in ("main_metrics", "same_point_stereo_reference", "changed_vs_stereo", "elapsed_seconds")}, ensure_ascii=False, indent=2))
|
|
if mode == "all":
|
|
from evaluate_streaming import refresh_streaming
|
|
streaming, _ = refresh_streaming("maai_user_only")
|
|
print('PRIMARY streaming metrics:', json.dumps(streaming['main_metrics']), flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("mode", choices=("smoke", "evaluate", "all"))
|
|
main(parser.parse_args().mode)
|