记录三模型话轮检测实验、44点评估及延迟对比
This commit is contained in:
Xin Wang
@@ -0,0 +1,5 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
.cache/
|
||||
models/
|
||||
*.wav
|
||||
@@ -0,0 +1 @@
|
||||
3.11
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Benchmark MaAI's incremental CPU updates with a full 20s context cache."""
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from smoke_run import HERE, ROOT, np, sf, soxr, torch
|
||||
from evaluate_can_reply import model
|
||||
|
||||
|
||||
def stats(values):
|
||||
v = np.asarray(values, dtype=float)
|
||||
return dict(n=len(v), mean_ms=float(v.mean()), median_ms=float(np.median(v)),
|
||||
p95_ms=float(np.percentile(v, 95)), p99_ms=float(np.percentile(v, 99)),
|
||||
min_ms=float(v.min()), max_ms=float(v.max()))
|
||||
|
||||
|
||||
def main():
|
||||
torch.set_num_threads(4)
|
||||
torch.set_num_interop_threads(1)
|
||||
with (ROOT / "labels/EOT标注/工作数据/eot_labels.csv").open() as f:
|
||||
labels = list(csv.DictReader(f))
|
||||
limits = {call: max(int(r["decision_ms"]) for r in labels if r["call_id"] == call)
|
||||
for call in sorted({r["call_id"] for r in labels})}
|
||||
prepared = []
|
||||
for call, limit in limits.items():
|
||||
pcm, sr = sf.read(ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3", dtype="float32", always_2d=True)
|
||||
audio = np.ascontiguousarray(soxr.resample(pcm[:limit*sr//1000], sr, 16000, quality="HQ"), dtype=np.float32)
|
||||
prepared.append((call, audio))
|
||||
measured, load_times = [], []
|
||||
for call, audio in prepared:
|
||||
began = time.perf_counter()
|
||||
maai = model()
|
||||
load_times.append((time.perf_counter()-began)*1000)
|
||||
maai.CALC_PROCESS_TIME_INTERVAL = 10**9
|
||||
current = {}
|
||||
original_encode, original_forward = maai.vap.encode_audio, maai.vap.forward
|
||||
|
||||
def encode(*args, **kwargs):
|
||||
began = time.perf_counter()
|
||||
output = original_encode(*args, **kwargs)
|
||||
current["encoder_ms"] = (time.perf_counter()-began)*1000
|
||||
return output
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
began = time.perf_counter()
|
||||
output = original_forward(*args, **kwargs)
|
||||
current["vap_ms"] = (time.perf_counter()-began)*1000
|
||||
return output
|
||||
|
||||
maai.vap.encode_audio, maai.vap.forward = encode, forward
|
||||
count = len(audio)//1600
|
||||
for frame in range(count):
|
||||
start = frame*1600
|
||||
ch1 = np.ascontiguousarray(audio[start:start+1600, 0])
|
||||
ch2 = np.ascontiguousarray(audio[start:start+1600, 1])
|
||||
current.clear()
|
||||
began = time.perf_counter()
|
||||
maai.process(ch1, ch2)
|
||||
result = maai.result_dict_queue.get_nowait()
|
||||
total = (time.perf_counter()-began)*1000
|
||||
assert np.isfinite(result["p_now"]).all()
|
||||
measured.append(dict(call_id=call, audio_end_sec=(frame+1)*0.1,
|
||||
full_context=int(frame >= 200), total_ms=total, **current))
|
||||
print(f"{call[:8]}: {count} frames, completed", flush=True)
|
||||
del maai
|
||||
warm = [r for r in measured if r["full_context"]]
|
||||
report = dict(method="Incremental process/get-result with 100ms stereo frames, 20s rolling KV cache. Steady-state excludes first 20s of each call. Includes CPC encoding, VAP forward, buffer/queue work; excludes imports, loading, decoding and resampling. No concurrent benchmark jobs.",
|
||||
cpu_threads=4, frame_rate=10, cache_seconds=20,
|
||||
model_load=stats(load_times),
|
||||
all_updates=stats([r["total_ms"] for r in measured]),
|
||||
steady_total=stats([r["total_ms"] for r in warm]),
|
||||
steady_encoder=stats([r["encoder_ms"] for r in warm]),
|
||||
steady_vap=stats([r["vap_ms"] for r in warm]),
|
||||
audio_seconds=len(measured)/10, samples=measured)
|
||||
point_rows = json.loads((HERE / "results/work-data-can-reply/summary.json").read_text())
|
||||
with (HERE / "results/work-data-can-reply/predictions.csv").open(encoding="utf-8-sig") as f:
|
||||
point_replay = list(csv.DictReader(f))
|
||||
report["previous_offline_point_replay"] = stats([float(r["replay_inference_ms"]) for r in point_replay])
|
||||
report["previous_point_eval_elapsed_seconds"] = point_rows["elapsed_seconds"]
|
||||
path = HERE / "results/latency.json"
|
||||
path.write_text(json.dumps(report, indent=2)+"\n")
|
||||
print(json.dumps({k:v for k,v in report.items() if k != "samples"}, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,170 @@
|
||||
"""Evaluate MaAI activity-projection proxies at exact annotated media times.
|
||||
|
||||
Each point uses a new model state and only preceding stereo audio, up to 20 s.
|
||||
Outputs are mapped to can_reply without any end-signal or timeout gating.
|
||||
"""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import contextlib
|
||||
import csv
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from smoke_run import HERE, ROOT, WEIGHTS, CPC, EXPECTED, Maai, MaaiInput, np, sf, soxr, torch
|
||||
|
||||
OUT = HERE / "results/work-data-can-reply"
|
||||
LABELS = ROOT / "labels/EOT标注/工作数据/eot_labels.csv"
|
||||
HISTORY = ROOT / "labels/EOT标注/工作数据/asr_history.csv"
|
||||
RULES = ("p_now_robot", "p_future_robot", "both_robot")
|
||||
|
||||
|
||||
def read(path):
|
||||
with path.open(encoding="utf-8-sig", newline="") as f:
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
|
||||
def write(path, rows):
|
||||
if not rows:
|
||||
return
|
||||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def metrics(rows, score="p_now_robot", threshold=0.5):
|
||||
counts = Counter((int(r["can_reply_now"]), int(float(r[score]) > threshold))
|
||||
for r in rows if r["can_reply_now"] in ("0", "1"))
|
||||
tn, fp, fn, tp = counts[0, 0], counts[0, 1], counts[1, 0], counts[1, 1]
|
||||
total = tn+fp+fn+tp
|
||||
ratio = lambda n, d: n/d if d else None
|
||||
return dict(n=total, tn=tn, fp=fp, fn=fn, tp=tp,
|
||||
accuracy=ratio(tn+tp, total), false_reply_rate=ratio(fp, fp+tn),
|
||||
false_hold_rate=ratio(fn, fn+tp), recall=ratio(tp, tp+fn),
|
||||
precision=ratio(tp, tp+fp), f1=ratio(2*tp, 2*tp+fp+fn))
|
||||
|
||||
|
||||
def model():
|
||||
# New instance also resets CPC recurrent state, not just VAP's KV cache.
|
||||
with contextlib.redirect_stdout(io.StringIO()), torch.serialization.safe_globals([argparse.Namespace]):
|
||||
return Maai(mode="vap", lang="ch_kyoto", audio_ch1=MaaiInput.Chunk(),
|
||||
audio_ch2=MaaiInput.Chunk(), frame_rate=10, context_len_sec=20,
|
||||
device="cpu", model_type="normal", mc=True, use_kv_cache=True,
|
||||
cpc_model=str(CPC), local_model=str(WEIGHTS),
|
||||
cache_dir=str(HERE / ".cache/huggingface"))
|
||||
|
||||
|
||||
def main():
|
||||
torch.set_num_threads(4)
|
||||
torch.set_num_interop_threads(1)
|
||||
label_hash = hashlib.sha256(LABELS.read_bytes()).hexdigest()
|
||||
for path in (WEIGHTS, CPC):
|
||||
assert hashlib.sha256(path.read_bytes()).hexdigest() == EXPECTED[path.name]
|
||||
labels, history = read(LABELS), read(HISTORY)
|
||||
keys = [(r["call_id"], r["decision_ms"]) for r in labels]
|
||||
assert len(set(keys)) == len(keys)
|
||||
previous_path = ROOT / "experiments/smart-turn-v3.2/results/work-data-can-reply/predictions.csv"
|
||||
previous_summary = json.loads(previous_path.with_name("summary.json").read_text())
|
||||
if previous_summary["config"]["label_sha256"] != label_hash:
|
||||
raise ValueError("Smart Turn baseline labels differ from current labels; rerun that baseline first")
|
||||
previous = {(r["call_id"], r["decision_ms"]): r for r in read(previous_path)}
|
||||
audio_cache, rows, audio_hashes = {}, [], {}
|
||||
began = time.perf_counter()
|
||||
for index, label in enumerate(labels):
|
||||
call, decision = label["call_id"], int(label["decision_ms"])
|
||||
if call not in audio_cache:
|
||||
path = ROOT / "data/韩泰机器人抢话/音频" / f"{call}.mp3"
|
||||
audio, sr = sf.read(path, dtype="float32", always_2d=True)
|
||||
assert audio.shape[1] == 2
|
||||
audio_cache[call] = audio, sr
|
||||
audio_hashes[call] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
audio, sr = audio_cache[call]
|
||||
# End aligned at decision_ms, so every point's final 100ms frame ends
|
||||
# exactly there. At file beginning drop <100ms from the leading edge.
|
||||
duration_ms = min(20000, decision // 100 * 100)
|
||||
if duration_ms == 0:
|
||||
raise ValueError(f"Insufficient history at {call} {decision}")
|
||||
origin = decision-duration_ms
|
||||
start, end = origin*sr//1000, decision*sr//1000
|
||||
assert 0 <= start < end <= len(audio)
|
||||
# Crop BEFORE resampling: resampler cannot observe future audio.
|
||||
segment = np.ascontiguousarray(soxr.resample(audio[start:end], sr, 16000, quality="HQ"), dtype=np.float32)
|
||||
count = duration_ms//100
|
||||
assert len(segment) == count*1600
|
||||
maai = model()
|
||||
maai.CALC_PROCESS_TIME_INTERVAL = 10**9
|
||||
inference_began = time.perf_counter()
|
||||
for frame in range(count):
|
||||
position = frame*1600
|
||||
maai.process(np.ascontiguousarray(segment[position:position+1600, 0]),
|
||||
np.ascontiguousarray(segment[position:position+1600, 1]))
|
||||
result = maai.result_dict_queue.get_nowait()
|
||||
assert maai.result_dict_queue.empty()
|
||||
inference_ms = (time.perf_counter()-inference_began)*1000
|
||||
values = {}
|
||||
for key in ("p_now", "p_future", "vad"):
|
||||
values[key] = np.asarray(result[key], dtype=float).reshape(-1)
|
||||
assert len(values[key]) == 2
|
||||
assert np.isfinite(values[key]).all() and ((values[key] >= 0) & (values[key] <= 1)).all()
|
||||
now, future = float(values["p_now"][1]), float(values["p_future"][1])
|
||||
expected = int(label["can_reply_now"])
|
||||
baseline = previous[(call, label["decision_ms"])]
|
||||
call_history = [r for r in history if r["call_id"] == call]
|
||||
agent_active = any(r["role"] == "agent" and int(r["start_ms"]) < decision < int(r["end_ms"]) for r in call_history)
|
||||
user_active = any(r["role"] == "user" and int(r["start_ms"]) < decision < int(r["end_ms"]) for r in call_history)
|
||||
r = dict(label)
|
||||
r.update(p_now_user=float(values["p_now"][0]), p_now_robot=now,
|
||||
p_future_user=float(values["p_future"][0]), p_future_robot=future,
|
||||
both_robot=min(now, future), vad_user=float(values["vad"][0]), vad_robot=float(values["vad"][1]),
|
||||
prediction_05=int(now > 0.5), correct_05=int(int(now > 0.5) == expected),
|
||||
prediction_08=int(now > 0.8), correct_08=int(int(now > 0.8) == expected),
|
||||
future_prediction_05=int(future > 0.5), both_prediction_05=int(min(now, future) > 0.5),
|
||||
smart_turn_probability=float(baseline["probability"]),
|
||||
smart_turn_prediction_05=int(baseline["prediction"]),
|
||||
smart_turn_prediction_08=int(float(baseline["probability"]) > 0.8),
|
||||
audio_start_ms=origin, audio_end_ms=decision, context_ms=duration_ms,
|
||||
frame_count=count, replay_inference_ms=inference_ms,
|
||||
asr_agent_active_at_point=int(agent_active), asr_user_segment_crosses_point=int(user_active),
|
||||
label_status="draft" if label["annotator"] == "xlsx_draft" else "unconfirmed_manual")
|
||||
rows.append(r)
|
||||
print(f"{index+1}/{len(labels)} {call[:8]} {decision}: label={expected}, p_now_robot={now:.4f}, p_future_robot={future:.4f}", flush=True)
|
||||
del maai
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
write(OUT / "predictions.csv", rows)
|
||||
write(OUT / "errors.csv", [r for r in rows if not r["correct_05"]])
|
||||
thresholds = [dict(rule=rule, threshold=threshold, **metrics(rows, rule, threshold))
|
||||
for rule in RULES for threshold in (0.5, 0.7, 0.8, 0.9, 0.95, 0.98)]
|
||||
write(OUT / "thresholds.csv", thresholds)
|
||||
groups = {field: {value: metrics([r for r in rows if r[field] == value])
|
||||
for value in sorted({r[field] for r in rows})}
|
||||
for field in ("call_id", "reason", "label_status", "asr_agent_active_at_point")}
|
||||
summary = dict(total_points=len(rows), main_rule="p_now_robot > 0.5", main_metrics=metrics(rows),
|
||||
rule_metrics={rule: metrics(rows, rule) for rule in RULES},
|
||||
thresholds=thresholds, grouped=groups,
|
||||
smart_turn_comparison={str(t): metrics(rows, "smart_turn_probability", t) for t in (0.5, 0.8)},
|
||||
changed_vs_smart_turn=dict(
|
||||
fixed=sum(int(r["smart_turn_prediction_05"]) != int(r["can_reply_now"]) and r["correct_05"] for r in rows),
|
||||
regressed=sum(int(r["smart_turn_prediction_05"]) == int(r["can_reply_now"]) and not r["correct_05"] for r in rows)),
|
||||
config=dict(maai_version=importlib.metadata.version("maai"), torch_version=torch.__version__,
|
||||
device="cpu", threads=4, frame_rate=10, context_len_sec=20, use_kv_cache=True,
|
||||
weight_variant="vap_mc", weight_sha256=EXPECTED, label_sha256=label_hash,
|
||||
history_sha256=hashlib.sha256(HISTORY.read_bytes()).hexdigest(), audio_sha256=audio_hashes,
|
||||
script_sha256=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()),
|
||||
elapsed_seconds=time.perf_counter()-began,
|
||||
method="At each exact decision_ms a fresh MaAI/CPC state replays a preceding <=20s stereo window in 100ms frames. Crop before resampling. No future input, no labels as model input, no EOT or VAD gates, no forced timeout replies. Classification is a stated proxy rule, not an official calibrated can_reply output.",
|
||||
limitations=["29 xlsx_draft and 15 rows without explicit verification metadata; exploratory selected-point scores.",
|
||||
"The proxy p_now[robot] forecasts relative speaker activity over 0–0.6s; p_future forecasts 0.6–2s. Neither directly predicts task completion.",
|
||||
"Independent 20s window restart differs from continuous whole-call state, particularly CPC recurrent state and deeply contextualized KV history.",
|
||||
"MaAI uses both recorded channels and up to20s; Smart Turn baseline uses user channel and up to8s. Comparison evaluates these input configurations, not identical-input model capability.",
|
||||
"Robot speech already present in recording can influence activity forecasts; agent-active checkpoints are separately marked.",
|
||||
"ASR alignment issues previously found at cb3238dc 13540 and e304f740 11531 remain; labels unchanged."])
|
||||
(OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2)+"\n")
|
||||
print(json.dumps({k:summary[k] for k in ("main_metrics", "rule_metrics", "changed_vs_smart_turn", "elapsed_seconds")}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
[project]
|
||||
name = "maai-ch-kyoto"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"maai==0.2.22",
|
||||
"soxr>=1.1.0",
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "tuna"
|
||||
url = "https://pypi.tuna.tsinghua.edu.cn/simple"
|
||||
default = true
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,12 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,p_now_user,p_now_robot,p_future_user,p_future_robot,both_robot,vad_user,vad_robot,prediction_05,correct_05,prediction_08,correct_08,future_prediction_05,both_prediction_05,smart_turn_probability,smart_turn_prediction_05,smart_turn_prediction_08,audio_start_ms,audio_end_ms,context_ms,frame_count,replay_inference_ms,asr_agent_active_at_point,asr_user_segment_crosses_point,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,0.48770105838775635,0.5122854709625244,0.3977898359298706,0.6022026538848877,0.5122854709625244,0.0005360433715395629,0.00010683283471735194,1,0,0,1,1,1,0.9848582744598389,1,1,28125,48125,20000,200,1017.8633751347661,0,1,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,0.29331472516059875,0.7066654562950134,0.29390087723731995,0.7060902714729309,0.7060902714729309,0.00018711105803959072,0.00024062590091489255,1,0,0,1,1,1,0.9729918241500854,1,1,56672,76672,20000,200,1007.945875171572,0,1,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,尽快，已经影响行车安全了。,1,low,completed_answer,xlsx_draft,0.885348916053772,0.11463910341262817,0.7168461680412292,0.28314584493637085,0.11463910341262817,0.022837858647108078,7.339265721384436e-05,0,0,0,0,0,0,0.9729918241500854,1,1,68482,88482,20000,200,1013.5022499598563,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,0.05483492836356163,0.9451597929000854,0.1947745531797409,0.8052172064781189,0.8052172064781189,0.0005812415038235486,0.9999233484268188,1,0,1,0,1,1,0.8732463121414185,1,1,50430,70430,20000,200,1008.9897499419749,1,1,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,82941,可以。,1,low,completed_answer,xlsx_draft,0.7286096811294556,0.27137291431427,0.720654308795929,0.27933770418167114,0.27137291431427,0.000260786444414407,8.787301339907572e-05,0,0,0,0,0,0,0.9801466464996338,1,1,62941,82941,20000,200,988.0605833604932,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,17153,我姓宋。,1,low,completed_answer,xlsx_draft,0.5991079807281494,0.4008772671222687,0.621396005153656,0.378596693277359,0.378596693277359,0.00047209489275701344,0.00011928172898478806,0,0,0,0,0,0,0.9880069494247437,1,1,53,17153,17100,171,824.6394577436149,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,0.19254903495311737,0.8074355721473694,0.2019786387681961,0.7980139255523682,0.7980139255523682,0.00010600832320051268,0.00039402887341566384,1,0,1,0,1,1,0.9889053702354431,1,1,48313,68313,20000,200,985.129208303988,0,1,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,29619,投诉。,1,low,completed_answer,xlsx_draft,0.5503987073898315,0.44958046078681946,0.5612668395042419,0.4387241005897522,0.4387241005897522,0.00015334364434238523,0.00010900569031946361,0,0,0,0,0,0,0.9604700207710266,1,1,9619,29619,20000,200,992.5864171236753,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,0.29946237802505493,0.700522243976593,0.3146718144416809,0.6853206753730774,0.6853206753730774,0.0006991042173467577,0.00023936097568366677,1,0,0,1,1,1,0.978554368019104,1,1,27187,47187,20000,200,999.0723747760057,0,1,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,0.0897170826792717,0.9102716445922852,0.12846434116363525,0.8715291023254395,0.8715291023254395,0.00023869278084021062,0.0010627142619341612,1,0,1,0,1,1,0.9870367050170898,1,1,21219,41219,20000,200,988.6126252822578,0,1,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,0.1399012804031372,0.8600831627845764,0.1313445121049881,0.8686479926109314,0.8600831627845764,0.00014179428399074823,0.0002719922340475023,1,0,1,0,1,1,0.9808988571166992,1,1,29204,49204,20000,200,1011.6221671923995,0,1,unconfirmed_manual
|
||||
|
@@ -0,0 +1,45 @@
|
||||
call_id,decision_ms,user_text_at_decision,can_reply_now,confidence,reason,annotator,p_now_user,p_now_robot,p_future_user,p_future_robot,both_robot,vad_user,vad_robot,prediction_05,correct_05,prediction_08,correct_08,future_prediction_05,both_prediction_05,smart_turn_probability,smart_turn_prediction_05,smart_turn_prediction_08,audio_start_ms,audio_end_ms,context_ms,frame_count,replay_inference_ms,asr_agent_active_at_point,asr_user_segment_crosses_point,label_status
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,21780,贵州省遵义市仁怀市。,1,low,completed_answer,xlsx_draft,0.04016409069299698,0.959829568862915,0.08882822096347809,0.9111647009849548,0.9111647009849548,0.00012105968198738992,0.9083148241043091,1,1,1,1,1,1,0.9875311851501465,1,1,1780,21780,20000,200,1053.8750002160668,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,29860,投诉建议。,1,low,completed_answer,xlsx_draft,0.07141099125146866,0.928583562374115,0.14475753903388977,0.8552355170249939,0.8552355170249939,8.126776810968295e-05,0.561092734336853,1,1,1,1,1,1,0.9403132200241089,1,1,9860,29860,20000,200,1007.4469172395766,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,型号是24年380大众帕萨特星空龙腾版。,0,low,mid_turn_pause,xlsx_draft,0.48770105838775635,0.5122854709625244,0.3977898359298706,0.6022026538848877,0.5122854709625244,0.0005360433715395629,0.00010683283471735194,1,0,0,1,1,1,0.9848582744598389,1,1,28125,48125,20000,200,1017.8633751347661,0,1,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,64000,行驶3万公里。,1,low,completed_answer,xlsx_draft,0.06301483511924744,0.9369784593582153,0.1267300695180893,0.8732624650001526,0.8732624650001526,0.00016355419938918203,0.6890804767608643,1,1,1,1,1,1,0.9875311851501465,1,1,44000,64000,20000,200,991.2372496910393,0,0,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,恒泰轮胎前门两条轮胎开裂,0,low,unfinished_expression,xlsx_draft,0.29331472516059875,0.7066654562950134,0.29390087723731995,0.7060902714729309,0.7060902714729309,0.00018711105803959072,0.00024062590091489255,1,0,0,1,1,1,0.9729918241500854,1,1,56672,76672,20000,200,1007.945875171572,0,1,draft
|
||||
2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,尽快，已经影响行车安全了。,1,low,completed_answer,xlsx_draft,0.885348916053772,0.11463910341262817,0.7168461680412292,0.28314584493637085,0.11463910341262817,0.022837858647108078,7.339265721384436e-05,0,0,0,0,0,0,0.9729918241500854,1,1,68482,88482,20000,200,1013.5022499598563,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,22007,宣传。,1,low,completed_answer,xlsx_draft,0.30675008893013,0.6932302713394165,0.36729562282562256,0.6326956152915955,0.6326956152915955,0.00010110728180734441,0.0004169040475971997,1,1,0,0,1,1,0.9865230321884155,1,1,2007,22007,20000,200,1012.2845829464495,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,31842,投诉建议。,1,low,completed_answer,xlsx_draft,0.2100573480129242,0.7899253964424133,0.23621736466884613,0.7637743949890137,0.7637743949890137,0.0003776845696847886,0.0002346531837247312,1,1,0,0,1,1,0.9889053702354431,1,1,11842,31842,20000,200,988.5724582709372,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,52348,爱车型号是大众帕萨特，购车时间2023年10月份，行驶里程24000多公里。,1,low,completed_answer,xlsx_draft,0.04021301865577698,0.9597742557525635,0.057571351528167725,0.9424217343330383,0.9424217343330383,7.617704977747053e-05,0.00035042857052758336,1,1,1,1,1,1,0.9889053702354431,1,1,32348,52348,20000,200,1034.5033751800656,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,轮胎裂纹。,0,low,mid_turn_pause,xlsx_draft,0.8353617191314697,0.16462112963199615,0.8176290392875671,0.18236291408538818,0.16462112963199615,0.0009962004842236638,9.046775085153058e-05,0,1,0,1,0,0,0.9777120351791382,1,1,45298,65298,20000,200,989.2037082463503,0,0,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,轮胎裂纹同意更换新轮胎。但是,0,low,mid_turn_pause,xlsx_draft,0.8745284080505371,0.1254596710205078,0.8549182415008545,0.1450749635696411,0.1254596710205078,0.00040660894592292607,7.286643085535616e-05,0,1,0,1,0,0,0.6701734066009521,1,0,47943,67943,20000,200,1061.6244589909911,0,1,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,轮胎裂纹同意更换新轮胎。但是20,0,low,unfinished_expression,xlsx_draft,0.05483492836356163,0.9451597929000854,0.1947745531797409,0.8052172064781189,0.8052172064781189,0.0005812415038235486,0.9999233484268188,1,0,1,0,1,1,0.8732463121414185,1,1,50430,70430,20000,200,1008.9897499419749,1,1,draft
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,82941,可以。,1,low,completed_answer,xlsx_draft,0.7286096811294556,0.27137291431427,0.720654308795929,0.27933770418167114,0.27137291431427,0.000260786444414407,8.787301339907572e-05,0,0,0,0,0,0,0.9801466464996338,1,1,62941,82941,20000,200,988.0605833604932,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,17153,我姓宋。,1,low,completed_answer,xlsx_draft,0.5991079807281494,0.4008772671222687,0.621396005153656,0.378596693277359,0.378596693277359,0.00047209489275701344,0.00011928172898478806,0,0,0,0,0,0,0.9880069494247437,1,1,53,17153,17100,171,824.6394577436149,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,24846,郑州市，河南郑州市。,1,low,completed_answer,xlsx_draft,0.30631449818611145,0.693669319152832,0.3508889675140381,0.6491031646728516,0.6491031646728516,0.00013861471961718053,0.00019757459813263267,1,1,0,0,1,1,0.9884648323059082,1,1,4846,24846,20000,200,995.0546673499048,0,0,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,33471,我的轮胎开裂了。,1,low,completed_answer,xlsx_draft,0.19791387021541595,0.8020712733268738,0.2569742798805237,0.7430181503295898,0.7430181503295898,5.471357508213259e-05,0.0002131979272235185,1,1,1,1,1,1,0.9854345321655273,1,1,13471,33471,20000,200,987.3237917199731,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,18027,天津市。,1,low,completed_answer,xlsx_draft,0.4226110577583313,0.5773740410804749,0.4604991376399994,0.5394934415817261,0.5394934415817261,0.0001245723251486197,0.00030579205485992134,1,1,0,0,1,1,0.9884648323059082,1,1,27,18027,18000,180,877.7980832383037,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,24768,投诉。,1,low,completed_answer,xlsx_draft,0.4403727054595947,0.5596095323562622,0.4655279517173767,0.5344639420509338,0.5344639420509338,0.00010676216334104538,0.00011399904906284064,1,1,0,0,1,1,0.9880069494247437,1,1,4768,24768,20000,200,990.6519157811999,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,迈腾2023款330豪华版2.0T。出现的问题是轮购车时间是2024年5月份，里程是6851公里。,0,low,unfinished_expression,xlsx_draft,0.9899334907531738,0.01005936972796917,0.8830422759056091,0.11694956570863724,0.01005936972796917,0.9938445091247559,3.428650234127417e-05,0,1,0,1,0,0,0.9333261251449585,1,1,31080,51080,20000,200,999.8775408603251,0,0,draft
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,轮胎质量问题规律，排水槽横向竖向龟裂。,0,low,mid_turn_pause,xlsx_draft,0.19254903495311737,0.8074355721473694,0.2019786387681961,0.7980139255523682,0.7980139255523682,0.00010600832320051268,0.00039402887341566384,1,0,1,0,1,1,0.9889053702354431,1,1,48313,68313,20000,200,985.129208303988,0,1,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,13540,辽宁葫芦岛。,1,low,completed_answer,xlsx_draft,0.05560389533638954,0.9443897008895874,0.12428648769855499,0.8757067918777466,0.8757067918777466,0.00041734895785339177,0.7873190641403198,1,1,1,1,1,1,0.9793655872344971,1,1,40,13540,13500,135,647.1840841695666,0,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,27520,投诉建议。,1,low,completed_answer,xlsx_draft,0.031801022589206696,0.9681935906410217,0.08646809309720993,0.9135249853134155,0.9135249853134155,3.0082404919085093e-05,0.9136934876441956,1,1,1,1,1,1,0.9870367050170898,1,1,7520,27520,20000,200,1033.3759170025587,0,0,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,60790,我的轮胎行驶28000公里。,0,low,unfinished_expression,xlsx_draft,0.9434071183204651,0.05658455193042755,0.9218999147415161,0.07809395343065262,0.05658455193042755,0.0005838724900968373,0.00014466057473327965,0,1,0,1,0,0,0.04772377014160156,0,0,40790,60790,20000,200,1001.5775831416249,0,1,draft
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,70745,轮胎开裂,0,low,mid_turn_pause,xlsx_draft,0.7112759351730347,0.2887146472930908,0.6814156174659729,0.31857794523239136,0.2887146472930908,0.0006039905129000545,0.00014655715494882315,0,1,0,1,0,0,0.9604700207710266,1,1,50745,70745,20000,200,990.8196250908077,0,1,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,18842,苏州昆山。,1,low,completed_answer,xlsx_draft,0.38070565462112427,0.6192749738693237,0.3990688920021057,0.6009228229522705,0.6009228229522705,8.244596392614767e-05,8.795281610218808e-05,1,1,0,0,1,1,0.8732463121414185,1,1,42,18842,18800,188,915.5570417642593,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,29619,投诉。,1,low,completed_answer,xlsx_draft,0.5503987073898315,0.44958046078681946,0.5612668395042419,0.4387241005897522,0.4387241005897522,0.00015334364434238523,0.00010900569031946361,0,0,0,0,0,0,0.9604700207710266,1,1,9619,29619,20000,200,992.5864171236753,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,40701,帕萨特,0,low,mid_turn_pause,xlsx_draft,0.5814271569252014,0.41855862736701965,0.6056566834449768,0.39433610439300537,0.39433610439300537,8.747771789785475e-05,0.00011669446394080296,0,1,0,1,0,0,0.9875311851501465,1,1,20701,40701,20000,200,1004.8157079145312,0,1,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,54600,24年11月份购买的帕萨特，15000公里左右。,1,low,completed_answer,xlsx_draft,0.29437172412872314,0.7056105136871338,0.31611114740371704,0.6838806867599487,0.6838806867599487,4.9861151637742296e-05,0.00015565029752906412,1,1,0,0,1,1,0.9170267581939697,1,1,34600,54600,20000,200,984.1025839559734,0,0,draft
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,65819,四条轮胎，最近我在。,0,low,unfinished_expression,xlsx_draft,0.94746994972229,0.0525195375084877,0.9264986515045166,0.07349509745836258,0.0525195375084877,0.00022494378208648413,5.3313040552893654e-05,0,1,0,1,0,0,0.01514175534248352,0,0,45819,65819,20000,200,985.2488748729229,0,1,draft
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,47187,帕萨特L2025款 ,0,,mid_turn_pause,,0.29946237802505493,0.700522243976593,0.3146718144416809,0.6853206753730774,0.6853206753730774,0.0006991042173467577,0.00023936097568366677,1,0,0,1,1,1,0.978554368019104,1,1,27187,47187,20000,200,999.0723747760057,0,1,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,52326,我的轮胎开裂了。,0,,mid_turn_pause,,0.5759003758430481,0.42408767342567444,0.5521559119224548,0.4478364586830139,0.42408767342567444,0.0029872211162000895,9.722011600388214e-05,0,1,0,1,0,0,0.9859892129898071,1,1,32326,52326,20000,200,1028.057583142072,0,1,unconfirmed_manual
|
||||
40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48348,爱车型号是大众帕萨特，购车时间2023年10月份,0,,mid_turn_pause,,0.6262152791023254,0.3737674355506897,0.4419881999492645,0.5580037236213684,0.3737674355506897,0.000293432705802843,8.914202044252306e-05,0,1,0,1,1,0,0.9889053702354431,1,1,28348,48348,20000,200,1016.654958948493,0,1,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,44810,帕萨特,0,,mid_turn_pause,,0.6299588680267334,0.37002745270729065,0.6284565925598145,0.3715363144874573,0.37002745270729065,0.0006757201626896858,0.0001261411962332204,0,1,0,1,0,0,0.9777120351791382,1,1,24810,44810,20000,200,1011.0269170254469,0,1,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,49500,帕萨特L2025款2025年3月购车,1,,completed_turn,,0.035902686417102814,0.9640915393829346,0.2039353996515274,0.7960534691810608,0.7960534691810608,0.002972200745716691,0.9997288584709167,1,1,1,1,1,1,0.9842595458030701,1,1,29500,49500,20000,200,1005.673292092979,0,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,54318,行驶里程29000公里。,1,,completed_turn,,0.06631098687648773,0.9336835145950317,0.13233250379562378,0.867659866809845,0.867659866809845,0.00030513154342770576,0.9997851252555847,1,1,1,1,1,1,0.9859892129898071,1,1,34318,54318,20000,200,991.5305417962372,0,0,unconfirmed_manual
|
||||
872a1e67-873a-4c49-a598-b38a670afcad,45581,帕萨特,0,,mid_turn_pause,,0.9767857193946838,0.023208266124129295,0.952804446220398,0.04718857631087303,0.023208266124129295,0.7301234006881714,7.874148286646232e-05,0,1,0,1,0,0,0.7923524379730225,1,0,25581,45581,20000,200,996.7834590934217,0,1,unconfirmed_manual
|
||||
a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,迈腾2023款330豪华版2.0T,0,,mid_turn_pause,,0.0897170826792717,0.9102716445922852,0.12846434116363525,0.8715291023254395,0.8715291023254395,0.00023869278084021062,0.0010627142619341612,1,0,1,0,1,1,0.9870367050170898,1,1,21219,41219,20000,200,988.6126252822578,0,1,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,19620,,1,,completed_turn,,0.02692226879298687,0.9730719327926636,0.09090150147676468,0.9090912342071533,0.9090912342071533,7.903630466898903e-05,0.9060366153717041,1,1,1,1,1,1,0.9870367050170898,1,1,20,19620,19600,196,976.409750059247,0,0,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,41207,我是2023年,0,,mid_turn_pause,,0.8767244219779968,0.12326544523239136,0.8488729000091553,0.15112045407295227,0.12326544523239136,0.00042212821426801383,0.00012722669634968042,0,1,0,1,0,0,0.9403132200241089,1,1,21207,41207,20000,200,1009.9681671708822,0,1,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,49204,我是2023年、2024年1月份购置的迈腾3302023款轿车,0,,mid_turn_pause,,0.1399012804031372,0.8600831627845764,0.1313445121049881,0.8686479926109314,0.8600831627845764,0.00014179428399074823,0.0002719922340475023,1,0,1,0,1,1,0.9808988571166992,1,1,29204,49204,20000,200,1011.6221671923995,0,1,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,58424,我的轮胎,0,,mid_turn_pause,,0.8062168955802917,0.19377247989177704,0.7670581936836243,0.23293526470661163,0.19377247989177704,0.00022283874568529427,9.903877071337774e-05,0,1,0,1,0,0,0.044268637895584106,0,0,38424,58424,20000,200,1021.3787080720067,0,1,unconfirmed_manual
|
||||
cb3238dc-3946-42a5-aad8-9b9a54226363,73640,轮胎开裂，行驶28000公里，轮胎开裂,0,,mid_turn_pause,,0.5417237281799316,0.45826253294944763,0.4124079644680023,0.5875844955444336,0.45826253294944763,0.0004774614062625915,0.00023703854822088033,0,1,0,1,1,0,0.9870367050170898,1,1,53640,73640,20000,200,1013.9890420250595,0,1,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,11531,姓赵,1,,completed_turn,,0.47997692227363586,0.5200050473213196,0.5343555808067322,0.46563640236854553,0.46563640236854553,9.148631943389773e-05,0.00014259465388022363,1,1,0,0,0,0,0.9880069494247437,1,1,31,11531,11500,115,518.2703747414052,0,0,unconfirmed_manual
|
||||
e304f740-85d4-427e-9fd6-c5474e9cd737,62828,四条轮胎,0,,mid_turn_pause,,0.9779409766197205,0.02204853855073452,0.9655047059059143,0.03448908030986786,0.02204853855073452,0.0003308310988359153,2.6734765924629755e-05,0,1,0,1,0,0,0.07174640893936157,0,0,42828,62828,20000,200,985.218208283186,0,1,unconfirmed_manual
|
||||
|
@@ -0,0 +1,589 @@
|
||||
{
|
||||
"total_points": 44,
|
||||
"main_rule": "p_now_robot > 0.5",
|
||||
"main_metrics": {
|
||||
"n": 44,
|
||||
"tn": 15,
|
||||
"fp": 7,
|
||||
"fn": 4,
|
||||
"tp": 18,
|
||||
"accuracy": 0.75,
|
||||
"false_reply_rate": 0.3181818181818182,
|
||||
"false_hold_rate": 0.18181818181818182,
|
||||
"recall": 0.8181818181818182,
|
||||
"precision": 0.72,
|
||||
"f1": 0.7659574468085106
|
||||
},
|
||||
"rule_metrics": {
|
||||
"p_now_robot": {
|
||||
"n": 44,
|
||||
"tn": 15,
|
||||
"fp": 7,
|
||||
"fn": 4,
|
||||
"tp": 18,
|
||||
"accuracy": 0.75,
|
||||
"false_reply_rate": 0.3181818181818182,
|
||||
"false_hold_rate": 0.18181818181818182,
|
||||
"recall": 0.8181818181818182,
|
||||
"precision": 0.72,
|
||||
"f1": 0.7659574468085106
|
||||
},
|
||||
"p_future_robot": {
|
||||
"n": 44,
|
||||
"tn": 13,
|
||||
"fp": 9,
|
||||
"fn": 5,
|
||||
"tp": 17,
|
||||
"accuracy": 0.6818181818181818,
|
||||
"false_reply_rate": 0.4090909090909091,
|
||||
"false_hold_rate": 0.22727272727272727,
|
||||
"recall": 0.7727272727272727,
|
||||
"precision": 0.6538461538461539,
|
||||
"f1": 0.7083333333333334
|
||||
},
|
||||
"both_robot": {
|
||||
"n": 44,
|
||||
"tn": 15,
|
||||
"fp": 7,
|
||||
"fn": 5,
|
||||
"tp": 17,
|
||||
"accuracy": 0.7272727272727273,
|
||||
"false_reply_rate": 0.3181818181818182,
|
||||
"false_hold_rate": 0.22727272727272727,
|
||||
"recall": 0.7727272727272727,
|
||||
"precision": 0.7083333333333334,
|
||||
"f1": 0.7391304347826086
|
||||
}
|
||||
},
|
||||
"thresholds": [
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.5,
|
||||
"n": 44,
|
||||
"tn": 15,
|
||||
"fp": 7,
|
||||
"fn": 4,
|
||||
"tp": 18,
|
||||
"accuracy": 0.75,
|
||||
"false_reply_rate": 0.3181818181818182,
|
||||
"false_hold_rate": 0.18181818181818182,
|
||||
"recall": 0.8181818181818182,
|
||||
"precision": 0.72,
|
||||
"f1": 0.7659574468085106
|
||||
},
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.7,
|
||||
"n": 44,
|
||||
"tn": 16,
|
||||
"fp": 6,
|
||||
"fn": 10,
|
||||
"tp": 12,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.2727272727272727,
|
||||
"false_hold_rate": 0.45454545454545453,
|
||||
"recall": 0.5454545454545454,
|
||||
"precision": 0.6666666666666666,
|
||||
"f1": 0.6
|
||||
},
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.8,
|
||||
"n": 44,
|
||||
"tn": 18,
|
||||
"fp": 4,
|
||||
"fn": 12,
|
||||
"tp": 10,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.18181818181818182,
|
||||
"false_hold_rate": 0.5454545454545454,
|
||||
"recall": 0.45454545454545453,
|
||||
"precision": 0.7142857142857143,
|
||||
"f1": 0.5555555555555556
|
||||
},
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.9,
|
||||
"n": 44,
|
||||
"tn": 20,
|
||||
"fp": 2,
|
||||
"fn": 13,
|
||||
"tp": 9,
|
||||
"accuracy": 0.6590909090909091,
|
||||
"false_reply_rate": 0.09090909090909091,
|
||||
"false_hold_rate": 0.5909090909090909,
|
||||
"recall": 0.4090909090909091,
|
||||
"precision": 0.8181818181818182,
|
||||
"f1": 0.5454545454545454
|
||||
},
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.95,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 17,
|
||||
"tp": 5,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 0.7727272727272727,
|
||||
"recall": 0.22727272727272727,
|
||||
"precision": 1.0,
|
||||
"f1": 0.37037037037037035
|
||||
},
|
||||
{
|
||||
"rule": "p_now_robot",
|
||||
"threshold": 0.98,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.5,
|
||||
"n": 44,
|
||||
"tn": 13,
|
||||
"fp": 9,
|
||||
"fn": 5,
|
||||
"tp": 17,
|
||||
"accuracy": 0.6818181818181818,
|
||||
"false_reply_rate": 0.4090909090909091,
|
||||
"false_hold_rate": 0.22727272727272727,
|
||||
"recall": 0.7727272727272727,
|
||||
"precision": 0.6538461538461539,
|
||||
"f1": 0.7083333333333334
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.7,
|
||||
"n": 44,
|
||||
"tn": 17,
|
||||
"fp": 5,
|
||||
"fn": 11,
|
||||
"tp": 11,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.22727272727272727,
|
||||
"false_hold_rate": 0.5,
|
||||
"recall": 0.5,
|
||||
"precision": 0.6875,
|
||||
"f1": 0.5789473684210527
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.8,
|
||||
"n": 44,
|
||||
"tn": 19,
|
||||
"fp": 3,
|
||||
"fn": 14,
|
||||
"tp": 8,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.13636363636363635,
|
||||
"false_hold_rate": 0.6363636363636364,
|
||||
"recall": 0.36363636363636365,
|
||||
"precision": 0.7272727272727273,
|
||||
"f1": 0.48484848484848486
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.9,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 18,
|
||||
"tp": 4,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 0.8181818181818182,
|
||||
"recall": 0.18181818181818182,
|
||||
"precision": 1.0,
|
||||
"f1": 0.3076923076923077
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.95,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
},
|
||||
{
|
||||
"rule": "p_future_robot",
|
||||
"threshold": 0.98,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.5,
|
||||
"n": 44,
|
||||
"tn": 15,
|
||||
"fp": 7,
|
||||
"fn": 5,
|
||||
"tp": 17,
|
||||
"accuracy": 0.7272727272727273,
|
||||
"false_reply_rate": 0.3181818181818182,
|
||||
"false_hold_rate": 0.22727272727272727,
|
||||
"recall": 0.7727272727272727,
|
||||
"precision": 0.7083333333333334,
|
||||
"f1": 0.7391304347826086
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.7,
|
||||
"n": 44,
|
||||
"tn": 17,
|
||||
"fp": 5,
|
||||
"fn": 11,
|
||||
"tp": 11,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.22727272727272727,
|
||||
"false_hold_rate": 0.5,
|
||||
"recall": 0.5,
|
||||
"precision": 0.6875,
|
||||
"f1": 0.5789473684210527
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.8,
|
||||
"n": 44,
|
||||
"tn": 19,
|
||||
"fp": 3,
|
||||
"fn": 14,
|
||||
"tp": 8,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.13636363636363635,
|
||||
"false_hold_rate": 0.6363636363636364,
|
||||
"recall": 0.36363636363636365,
|
||||
"precision": 0.7272727272727273,
|
||||
"f1": 0.48484848484848486
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.9,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 18,
|
||||
"tp": 4,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 0.8181818181818182,
|
||||
"recall": 0.18181818181818182,
|
||||
"precision": 1.0,
|
||||
"f1": 0.3076923076923077
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.95,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
},
|
||||
{
|
||||
"rule": "both_robot",
|
||||
"threshold": 0.98,
|
||||
"n": 44,
|
||||
"tn": 22,
|
||||
"fp": 0,
|
||||
"fn": 22,
|
||||
"tp": 0,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 1.0,
|
||||
"recall": 0.0,
|
||||
"precision": null,
|
||||
"f1": 0.0
|
||||
}
|
||||
],
|
||||
"grouped": {
|
||||
"call_id": {
|
||||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||||
"n": 6,
|
||||
"tn": 0,
|
||||
"fp": 2,
|
||||
"fn": 1,
|
||||
"tp": 3,
|
||||
"accuracy": 0.5,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": 0.25,
|
||||
"recall": 0.75,
|
||||
"precision": 0.6,
|
||||
"f1": 0.6666666666666666
|
||||
},
|
||||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||||
"n": 8,
|
||||
"tn": 3,
|
||||
"fp": 1,
|
||||
"fn": 1,
|
||||
"tp": 3,
|
||||
"accuracy": 0.75,
|
||||
"false_reply_rate": 0.25,
|
||||
"false_hold_rate": 0.25,
|
||||
"recall": 0.75,
|
||||
"precision": 0.75,
|
||||
"f1": 0.75
|
||||
},
|
||||
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||||
"n": 9,
|
||||
"tn": 3,
|
||||
"fp": 1,
|
||||
"fn": 1,
|
||||
"tp": 4,
|
||||
"accuracy": 0.7777777777777778,
|
||||
"false_reply_rate": 0.25,
|
||||
"false_hold_rate": 0.2,
|
||||
"recall": 0.8,
|
||||
"precision": 0.8,
|
||||
"f1": 0.8
|
||||
},
|
||||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||||
"n": 5,
|
||||
"tn": 1,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 2,
|
||||
"accuracy": 0.6,
|
||||
"false_reply_rate": 0.6666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5,
|
||||
"f1": 0.6666666666666666
|
||||
},
|
||||
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||||
"n": 9,
|
||||
"tn": 5,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 3,
|
||||
"accuracy": 0.8888888888888888,
|
||||
"false_reply_rate": 0.16666666666666666,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.75,
|
||||
"f1": 0.8571428571428571
|
||||
},
|
||||
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||||
"n": 7,
|
||||
"tn": 3,
|
||||
"fp": 0,
|
||||
"fn": 1,
|
||||
"tp": 3,
|
||||
"accuracy": 0.8571428571428571,
|
||||
"false_reply_rate": 0.0,
|
||||
"false_hold_rate": 0.25,
|
||||
"recall": 0.75,
|
||||
"precision": 1.0,
|
||||
"f1": 0.8571428571428571
|
||||
}
|
||||
},
|
||||
"reason": {
|
||||
"completed_answer": {
|
||||
"n": 18,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 4,
|
||||
"tp": 14,
|
||||
"accuracy": 0.7777777777777778,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.2222222222222222,
|
||||
"recall": 0.7777777777777778,
|
||||
"precision": 1.0,
|
||||
"f1": 0.875
|
||||
},
|
||||
"completed_turn": {
|
||||
"n": 4,
|
||||
"tn": 0,
|
||||
"fp": 0,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 1.0,
|
||||
"false_reply_rate": null,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 1.0,
|
||||
"f1": 1.0
|
||||
},
|
||||
"mid_turn_pause": {
|
||||
"n": 17,
|
||||
"tn": 12,
|
||||
"fp": 5,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.7058823529411765,
|
||||
"false_reply_rate": 0.29411764705882354,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
},
|
||||
"unfinished_expression": {
|
||||
"n": 5,
|
||||
"tn": 3,
|
||||
"fp": 2,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.6,
|
||||
"false_reply_rate": 0.4,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
}
|
||||
},
|
||||
"label_status": {
|
||||
"draft": {
|
||||
"n": 29,
|
||||
"tn": 7,
|
||||
"fp": 4,
|
||||
"fn": 4,
|
||||
"tp": 14,
|
||||
"accuracy": 0.7241379310344828,
|
||||
"false_reply_rate": 0.36363636363636365,
|
||||
"false_hold_rate": 0.2222222222222222,
|
||||
"recall": 0.7777777777777778,
|
||||
"precision": 0.7777777777777778,
|
||||
"f1": 0.7777777777777778
|
||||
},
|
||||
"unconfirmed_manual": {
|
||||
"n": 15,
|
||||
"tn": 8,
|
||||
"fp": 3,
|
||||
"fn": 0,
|
||||
"tp": 4,
|
||||
"accuracy": 0.8,
|
||||
"false_reply_rate": 0.2727272727272727,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5714285714285714,
|
||||
"f1": 0.7272727272727273
|
||||
}
|
||||
},
|
||||
"asr_agent_active_at_point": {
|
||||
"0": {
|
||||
"n": 43,
|
||||
"tn": 15,
|
||||
"fp": 6,
|
||||
"fn": 4,
|
||||
"tp": 18,
|
||||
"accuracy": 0.7674418604651163,
|
||||
"false_reply_rate": 0.2857142857142857,
|
||||
"false_hold_rate": 0.18181818181818182,
|
||||
"recall": 0.8181818181818182,
|
||||
"precision": 0.75,
|
||||
"f1": 0.782608695652174
|
||||
},
|
||||
"1": {
|
||||
"n": 1,
|
||||
"tn": 0,
|
||||
"fp": 1,
|
||||
"fn": 0,
|
||||
"tp": 0,
|
||||
"accuracy": 0.0,
|
||||
"false_reply_rate": 1.0,
|
||||
"false_hold_rate": null,
|
||||
"recall": null,
|
||||
"precision": 0.0,
|
||||
"f1": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"smart_turn_comparison": {
|
||||
"0.5": {
|
||||
"n": 44,
|
||||
"tn": 4,
|
||||
"fp": 18,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.5909090909090909,
|
||||
"false_reply_rate": 0.8181818181818182,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.55,
|
||||
"f1": 0.7096774193548387
|
||||
},
|
||||
"0.8": {
|
||||
"n": 44,
|
||||
"tn": 6,
|
||||
"fp": 16,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.6363636363636364,
|
||||
"false_reply_rate": 0.7272727272727273,
|
||||
"false_hold_rate": 0.0,
|
||||
"recall": 1.0,
|
||||
"precision": 0.5789473684210527,
|
||||
"f1": 0.7333333333333333
|
||||
}
|
||||
},
|
||||
"changed_vs_smart_turn": {
|
||||
"fixed": 11,
|
||||
"regressed": 4
|
||||
},
|
||||
"config": {
|
||||
"maai_version": "0.2.22",
|
||||
"torch_version": "2.14.1",
|
||||
"device": "cpu",
|
||||
"threads": 4,
|
||||
"frame_rate": 10,
|
||||
"context_len_sec": 20,
|
||||
"use_kv_cache": true,
|
||||
"weight_variant": "vap_mc",
|
||||
"weight_sha256": {
|
||||
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
||||
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
||||
},
|
||||
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||||
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
||||
"audio_sha256": {
|
||||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
||||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
||||
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
||||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
||||
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
||||
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
||||
},
|
||||
"script_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe"
|
||||
},
|
||||
"elapsed_seconds": 45.435372250154614,
|
||||
"method": "At each exact decision_ms a fresh MaAI/CPC state replays a preceding <=20s stereo window in 100ms frames. Crop before resampling. No future input, no labels as model input, no EOT or VAD gates, no forced timeout replies. Classification is a stated proxy rule, not an official calibrated can_reply output.",
|
||||
"limitations": [
|
||||
"29 xlsx_draft and 15 rows without explicit verification metadata; exploratory selected-point scores.",
|
||||
"The proxy p_now[robot] forecasts relative speaker activity over 0–0.6s; p_future forecasts 0.6–2s. Neither directly predicts task completion.",
|
||||
"Independent 20s window restart differs from continuous whole-call state, particularly CPC recurrent state and deeply contextualized KV history.",
|
||||
"MaAI uses both recorded channels and up to20s; Smart Turn baseline uses user channel and up to8s. Comparison evaluates these input configurations, not identical-input model capability.",
|
||||
"Robot speech already present in recording can influence activity forecasts; agent-active checkpoints are separately marked.",
|
||||
"ASR alignment issues previously found at cb3238dc 13540 and e304f740 11531 remain; labels unchanged."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
rule,threshold,n,tn,fp,fn,tp,accuracy,false_reply_rate,false_hold_rate,recall,precision,f1
|
||||
p_now_robot,0.5,44,15,7,4,18,0.75,0.3181818181818182,0.18181818181818182,0.8181818181818182,0.72,0.7659574468085106
|
||||
p_now_robot,0.7,44,16,6,10,12,0.6363636363636364,0.2727272727272727,0.45454545454545453,0.5454545454545454,0.6666666666666666,0.6
|
||||
p_now_robot,0.8,44,18,4,12,10,0.6363636363636364,0.18181818181818182,0.5454545454545454,0.45454545454545453,0.7142857142857143,0.5555555555555556
|
||||
p_now_robot,0.9,44,20,2,13,9,0.6590909090909091,0.09090909090909091,0.5909090909090909,0.4090909090909091,0.8181818181818182,0.5454545454545454
|
||||
p_now_robot,0.95,44,22,0,17,5,0.6136363636363636,0.0,0.7727272727272727,0.22727272727272727,1.0,0.37037037037037035
|
||||
p_now_robot,0.98,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
p_future_robot,0.5,44,13,9,5,17,0.6818181818181818,0.4090909090909091,0.22727272727272727,0.7727272727272727,0.6538461538461539,0.7083333333333334
|
||||
p_future_robot,0.7,44,17,5,11,11,0.6363636363636364,0.22727272727272727,0.5,0.5,0.6875,0.5789473684210527
|
||||
p_future_robot,0.8,44,19,3,14,8,0.6136363636363636,0.13636363636363635,0.6363636363636364,0.36363636363636365,0.7272727272727273,0.48484848484848486
|
||||
p_future_robot,0.9,44,22,0,18,4,0.5909090909090909,0.0,0.8181818181818182,0.18181818181818182,1.0,0.3076923076923077
|
||||
p_future_robot,0.95,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
p_future_robot,0.98,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
both_robot,0.5,44,15,7,5,17,0.7272727272727273,0.3181818181818182,0.22727272727272727,0.7727272727272727,0.7083333333333334,0.7391304347826086
|
||||
both_robot,0.7,44,17,5,11,11,0.6363636363636364,0.22727272727272727,0.5,0.5,0.6875,0.5789473684210527
|
||||
both_robot,0.8,44,19,3,14,8,0.6136363636363636,0.13636363636363635,0.6363636363636364,0.36363636363636365,0.7272727272727273,0.48484848484848486
|
||||
both_robot,0.9,44,22,0,18,4,0.5909090909090909,0.0,0.8181818181818182,0.18181818181818182,1.0,0.3076923076923077
|
||||
both_robot,0.95,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
both_robot,0.98,44,22,0,22,0,0.5,0.0,1.0,0.0,,0.0
|
||||
|
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"rule": "1 - p_now[user] > 0.11",
|
||||
"threshold": 0.11,
|
||||
"metrics": {
|
||||
"n": 44,
|
||||
"tn": 5,
|
||||
"fp": 17,
|
||||
"fn": 0,
|
||||
"tp": 22,
|
||||
"accuracy": 0.6136363636363636,
|
||||
"false_reply_rate": 0.7727272727272727,
|
||||
"false_hold_rate": 0.0
|
||||
},
|
||||
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||||
"selection": "Supplementary operating point selected on these same 44 labels; not an independent validation or a guarantee of zero false holds on new audio.",
|
||||
"lowest_positive": {
|
||||
"call_id": "2c83253a-7977-4e3c-a387-0aa1ea06fbe1",
|
||||
"decision_ms": "88482",
|
||||
"score": 0.11465108394622803,
|
||||
"text": "尽快，已经影响行车安全了。"
|
||||
},
|
||||
"model_inference_unchanged": true,
|
||||
"same_predictions_using_p_now_robot": true
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
MaAI ch_kyoto 在工作数据判断点上的 can_reply 映射评估
|
||||
日期：2026-10-13
|
||||
|
||||
映射规则
|
||||
主要结果采用 p_now[1] > 0.5：把“机器人在未来约0–0.6秒的相对活动占比”映射为可回复。
|
||||
另外评估 p_future[1] > 0.5（约0.6–2秒），以及 p_now[1] 和 p_future[1] 同时超过0.5。
|
||||
这些是本次明确设定的代理判断规则，不是 MaAI 官方的 can_reply 校准输出。没有加入 VAD 停止门槛、结束信号或超时强制放行。
|
||||
|
||||
输入与运行
|
||||
MaAI 0.2.22，ch_kyoto 降噪 VAP（CPC编码器），CPU，4线程，10 Hz，20秒上下文，KV cache。
|
||||
每个 decision_ms 从其之前裁取最多20秒的双声道音频，先裁剪后重采样到16kHz；第1声道用户、第2声道机器人。
|
||||
窗口以判断点结束，长度为100毫秒整数倍，最终一帧准确结束在 decision_ms，没有使用判断点之后的音频。不足20秒时，仅丢弃录音开头不足100毫秒的残段。
|
||||
每个点重新构造模型，重置CPC循环状态和VAP缓存，避免不同点互相污染。该方式不同于从录音起点连续运行到判断时刻，且与Smart Turn的用户单声道/最多8秒上下文不同。
|
||||
44个点全部跑过真实模型推理，共耗时约45.4秒（包含逐点重载和重放）。原始人工标注未修改。
|
||||
|
||||
主要结果：p_now[1] > 0.5
|
||||
准确率：33/44 = 75.00%。
|
||||
人工允许接话22个：正确放行18个，误阻止4个，检出率81.82%。
|
||||
人工要求等待22个：正确等待15个，误放行7个，误放行率31.82%。
|
||||
放行精确率：18/25 = 72.00%。F1 = 0.7660。
|
||||
|
||||
与已有Smart Turn结果对照
|
||||
Smart Turn阈值0.5：准确率59.09%，误放行18/22，误阻止0/22。
|
||||
MaAI映射阈值0.5：准确率75.00%，误放行7/22，误阻止4/22。
|
||||
MaAI修正了11个Smart Turn误放行点，新增4个误阻止点。
|
||||
两个模型使用相同的44个判断点和人工标签，但输入声道、上下文长度和输出任务不同。对照仅说明这两种配置在当前选定点上的表现。
|
||||
|
||||
阈值0.8
|
||||
MaAI：正确28/44（63.64%），误放行4/22（18.18%），误阻止12/22（54.55%）。
|
||||
Smart Turn：正确28/44（63.64%），误放行16/22（72.73%），误阻止0/22。
|
||||
两者准确率相同，但错误分布明显不同；将相同数字阈值套用到不同输出分数并不代表相同工作点。
|
||||
|
||||
其他代理规则：阈值均0.5
|
||||
p_future[1]：准确率68.18%，误放行9个，误阻止5个。
|
||||
两个窗口同时通过：准确率72.73%，误放行7个，误阻止5个。
|
||||
这些对照在同一批标签上进行，不能据此当作独立测试集选参结果。
|
||||
|
||||
补充方案：当前标注点误阻止为0（2026-10-14）
|
||||
沿用TurnBench的分数映射：score = 1 - p_now[用户]，can_reply = score > 0.11。
|
||||
在这44个点中，与p_now[机器人] > 0.11的判断完全一致；原来的0.5主方案保留。
|
||||
准确率27/44（61.36%），误放行17/22（77.27%），误阻止0/22（0%），正确等待5/22、正确放行22/22。
|
||||
相比0.5阈值，修正原先4个误阻止，同时新增10个误放行。
|
||||
可回复点最低分为0.11465108394622803，来自2c83253a的88.482秒“尽快，已经影响行车安全了。”；0.12阈值会重新误阻止这个点。
|
||||
同为误阻止0的工作点：Smart Turn 0.5误放行18点，MaAI 0.11误放行17点，Smart Turn 0.8误放行16点。
|
||||
这里只重新映射已保存的模型输出，不改模型、输入、上下文或推理流程，已测量的计算延迟不变。
|
||||
该阈值使用当前标签选择，不是独立验证，不保证新数据仍为零误阻止。详细配置见zero-false-hold.json；三模型对比已同步更新。
|
||||
|
||||
错误分组与样例
|
||||
17个 mid_turn_pause 中误放行5个，相比Smart Turn的15个减少10个。
|
||||
5个 unfinished_expression 中误放行2个，相比Smart Turn的3个减少1个。
|
||||
正确拦截示例：872a1e67在44.810秒“帕萨特”，p_now机器人分数0.3700；40ae686c在48.348秒车型/购车时间组合回答，分数0.3738；e304f740在40.701秒“帕萨特”，分数0.4186。
|
||||
仍误放行示例：a266b9ef在41.219秒“迈腾2023款330豪华版2.0T”，分数0.9103；cb3238dc在49.204秒车型/购车时间表达，分数0.8601。
|
||||
误阻止4个：
|
||||
2c83253a 88.482秒“尽快，已经影响行车安全了。”，分数0.1146。
|
||||
40ae686c 82.941秒“可以。”，分数0.2714。
|
||||
872a1e67 17.153秒“我姓宋。”，分数0.4009。
|
||||
e304f740 29.619秒“投诉。”，分数0.4496。
|
||||
这说明该代理规则对中间停顿更谨慎，同时会拒绝部分人工已判完成的回答；需回听核对其语气和判断点时刻。
|
||||
|
||||
重要数据边界
|
||||
40ae686c的70.430秒，原录音机器人已经在讲话，p_now机器人分数0.9452，人工标签为等待。MaAI是在预测“谁会说话”，已发生的机器人插话会影响这一代理分数；不能将该分数直接解释为插话是否合适。
|
||||
排除该机器人正在发声的点后，其余43点正确33个（76.74%），误放行6/21，误阻止4/22。
|
||||
29个标签仍为xlsx_draft，另外15个未记录标注员/置信度，没有明确回听确认状态。结果属于探索性点级比较，不是正式验收准确率或静音区间抢话率。
|
||||
先前发现的cb3238dc 13.540秒文本/转写时间冲突、e304f740 11.531秒用户转写缺失仍然保留，不修改人工标签。
|
||||
|
||||
输出
|
||||
predictions.csv：44点的p_now、p_future、vad，0.5/0.8预测、Smart Turn对照、音频窗口和数据标记。
|
||||
errors.csv：主要规则的11个错误点。
|
||||
thresholds.csv：三个代理规则在多个阈值下的诊断指标。
|
||||
summary.json：完整统计、版本、参数、模型/代码/音频/标注SHA256和评估口径。
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Run MaAI ch_kyoto synchronously on a local stereo recording, on CPU."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.metadata
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parents[1]
|
||||
os.environ.setdefault("HF_HOME", str(HERE / ".cache/huggingface"))
|
||||
os.environ.setdefault("TORCH_HOME", str(HERE / ".cache/torch"))
|
||||
os.environ.setdefault("MPLCONFIGDIR", str(HERE / ".cache/matplotlib"))
|
||||
os.environ.setdefault("PYGAME_HIDE_SUPPORT_PROMPT", "1")
|
||||
os.environ.setdefault("HF_HUB_OFFLINE", "1")
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import soxr
|
||||
import torch
|
||||
from maai import Maai, MaaiInput
|
||||
|
||||
WEIGHTS = HERE / "models/vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt"
|
||||
CPC = HERE / "models/60k_epoch4-d0f474de.pt"
|
||||
EXPECTED = {
|
||||
WEIGHTS.name: "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
||||
CPC.name: "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--audio", type=Path, default=ROOT / "data/韩泰机器人抢话/音频/2c83253a-7977-4e3c-a387-0aa1ea06fbe1.mp3")
|
||||
parser.add_argument("--seconds", type=float, default=22)
|
||||
parser.add_argument("--threads", type=int, default=4)
|
||||
parser.add_argument("--output", type=Path, default=HERE / "results/smoke.json")
|
||||
args = parser.parse_args()
|
||||
if args.seconds <= 0 or args.threads < 1:
|
||||
parser.error("seconds and threads must be positive")
|
||||
hashes = {}
|
||||
for path in (WEIGHTS, CPC):
|
||||
if not path.exists():
|
||||
raise FileNotFoundError(f"Download the official weights first: {path}")
|
||||
hashes[path.name] = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
if hashes[path.name] != EXPECTED[path.name]:
|
||||
raise ValueError(f"Weight checksum mismatch: {path}")
|
||||
torch.set_num_threads(args.threads)
|
||||
torch.set_num_interop_threads(1)
|
||||
audio, original_sr = sf.read(args.audio, dtype="float32", always_2d=True)
|
||||
if audio.shape[1] != 2:
|
||||
raise ValueError("Expected stereo audio: channel 0 user, channel 1 robot")
|
||||
audio = audio[:round(args.seconds * original_sr)]
|
||||
if original_sr != 16000:
|
||||
audio = soxr.resample(audio, original_sr, 16000, quality="HQ")
|
||||
audio = np.ascontiguousarray(audio, dtype=np.float32)
|
||||
|
||||
load_started = time.perf_counter()
|
||||
# Official CPC checkpoint contains an argparse.Namespace. Allow only
|
||||
# that class for this load, while retaining weights_only safe loading.
|
||||
with torch.serialization.safe_globals([argparse.Namespace]):
|
||||
model = Maai(
|
||||
mode="vap", lang="ch_kyoto",
|
||||
audio_ch1=MaaiInput.Chunk(), audio_ch2=MaaiInput.Chunk(),
|
||||
frame_rate=10, context_len_sec=20, device="cpu",
|
||||
model_type="normal", mc=True, use_kv_cache=True,
|
||||
cpc_model=str(CPC), local_model=str(WEIGHTS),
|
||||
cache_dir=str(HERE / ".cache/huggingface"),
|
||||
)
|
||||
load_seconds = time.perf_counter() - load_started
|
||||
frame_samples = model.audio_samples_per_frame
|
||||
frame_count = len(audio) // frame_samples
|
||||
if frame_count < 1:
|
||||
raise ValueError("Audio must contain at least one 100 ms frame")
|
||||
frames, latencies = [], []
|
||||
inference_started = time.perf_counter()
|
||||
for i in range(frame_count):
|
||||
start = i*frame_samples
|
||||
before = time.perf_counter()
|
||||
model.process(np.ascontiguousarray(audio[start:start+frame_samples, 0]),
|
||||
np.ascontiguousarray(audio[start:start+frame_samples, 1]))
|
||||
latency_ms = (time.perf_counter() - before) * 1000
|
||||
# A complete CPC frame should always produce exactly one result.
|
||||
# Avoid blocking indefinitely if that assumption fails.
|
||||
result = model.result_dict_queue.get_nowait()
|
||||
assert model.result_dict_queue.empty(), "More than one result per frame"
|
||||
row = dict(audio_end_sec=(i+1)*frame_samples/16000,
|
||||
inference_ms=latency_ms)
|
||||
for key in ("p_now", "p_future", "vad"):
|
||||
values = np.asarray(result[key], dtype=np.float64).reshape(-1)
|
||||
if len(values) != 2 or not np.isfinite(values).all():
|
||||
raise ValueError(f"Invalid {key}: {values}")
|
||||
if not ((values >= 0) & (values <= 1)).all():
|
||||
raise ValueError(f"Out-of-range {key}: {values}")
|
||||
row[key] = values.tolist()
|
||||
frames.append(row)
|
||||
latencies.append(latency_ms)
|
||||
inference_seconds = time.perf_counter() - inference_started
|
||||
audio_seconds = frame_count*frame_samples/16000
|
||||
warm = latencies[min(5, len(latencies)-1):]
|
||||
report = dict(
|
||||
status="success", audio=str(args.audio.resolve()),
|
||||
original_sample_rate=original_sr, model_sample_rate=16000,
|
||||
audio_seconds=audio_seconds, frame_count=frame_count,
|
||||
model="ch_kyoto noise-robust VAP / CPC", weight_variant="vap_mc",
|
||||
maai_version=importlib.metadata.version("maai"), torch_version=torch.__version__,
|
||||
portaudio_dependency="Homebrew PortAudio 19.7.0 / PyAudio 0.2.14",
|
||||
device="cpu", threads=args.threads, frame_rate=10, context_len_sec=20,
|
||||
use_kv_cache=True, model_load_seconds=load_seconds,
|
||||
inference_seconds=inference_seconds, realtime_factor=inference_seconds/audio_seconds,
|
||||
warm_mean_frame_ms=float(np.mean(warm)), warm_p95_frame_ms=float(np.percentile(warm, 95)),
|
||||
weight_sha256=hashes,
|
||||
output_definitions={
|
||||
"channels": "Index 0=user, index 1=robot; separate stereo channels, no mixing.",
|
||||
"p_now": "Relative activity projection for the two speakers over roughly the next 0–0.6 seconds.",
|
||||
"p_future": "Relative activity projection for the two speakers over roughly the next 0.6–2 seconds.",
|
||||
"vad": "Current voice-activity probability for each speaker.",
|
||||
"audio_end_sec": "Media time measured from file start; MaAI wall-clock result t is not used.",
|
||||
"scope": "Basic offline CPU smoke test; VAP projections are not can_reply or EOT probabilities. Timings exclude model loading and resampling and describe this one recording/run only.",
|
||||
}, frames=frames)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, ensure_ascii=False, indent=2)+"\n")
|
||||
print(json.dumps({k:v for k,v in report.items() if k not in ("frames", "output_definitions", "weight_sha256")}, ensure_ascii=False, indent=2))
|
||||
print("Last frame:", json.dumps(frames[-1], ensure_ascii=False))
|
||||
print(f"Saved {args.output.resolve()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Generated
+2436
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,48 @@
|
||||
MaAI ch_kyoto 本机 CPU 基本运行
|
||||
验证日期：2026-10-13
|
||||
|
||||
环境
|
||||
Apple M4 Mac mini，16 GB 内存，原生 ARM64 Python 3.11.15。
|
||||
MaAI 0.2.22，PyTorch 2.14.1，PortAudio 19.7.0，PyAudio 0.2.14。
|
||||
uv 管理独立 .venv 和 uv.lock；PyPI 使用清华镜像。
|
||||
|
||||
复跑
|
||||
cd "/Users/wangx/Code/晓白灵智能打断/experiments/maai-ch-kyoto"
|
||||
uv run --locked python smoke_run.py
|
||||
|
||||
指定录音与时长
|
||||
uv run --locked python smoke_run.py --audio "/录音绝对路径.mp3" --seconds 22 --threads 4
|
||||
|
||||
输入须为双声道：索引 0 用户、索引 1 机器人。脚本读取头部指定时长，重采样至 16 kHz；按两路各 1600 样本（100 毫秒）同步调用 process，不启用麦克风或播放音频。末尾不足 100 毫秒的样本不参与推理。
|
||||
|
||||
模型
|
||||
语言 ch_kyoto，降噪 VAP 权重（vap_mc），原版 CPC 编码器（model_type=normal）。
|
||||
10 Hz，20 秒滚动上下文，KV cache，device=cpu。
|
||||
两个权重均存储在 models/，脚本逐次验证 SHA256 后离线加载。
|
||||
VAP 来源：https://huggingface.co/maai-kyoto/vap_ch_kyoto
|
||||
VAP 文件：vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt
|
||||
VAP SHA256：65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8
|
||||
CPC 来源：https://dl.fbaipublicfiles.com/librilight/CPC_checkpoints/60k_epoch4-d0f474de.pt
|
||||
CPC SHA256：d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1
|
||||
下载 VAP 时使用 hf-mirror，已核对官方完整 SHA256；CPC 从官方地址下载，并核对文件名哈希前缀及保存完整 SHA256。
|
||||
|
||||
验证结果
|
||||
录音：2c83253a-7977-4e3c-a387-0aa1ea06fbe1.mp3 前 22 秒。
|
||||
220 帧输出，所有 p_now / p_future / vad 均为两路有限、0–1 范围数值。
|
||||
模型加载约 0.083 秒，推理约 1.142 秒，RTF 0.052。
|
||||
跳过前 5 帧后，平均每帧 5.07 毫秒，P95 5.93 毫秒。
|
||||
计时不包含录音读取、重采样和模型加载；仅代表该录音的一次离线 CPU 测试。
|
||||
结果位于 results/smoke.json，包括配置、权重校验、逐帧媒体时间及概率。
|
||||
|
||||
输出含义
|
||||
p_now：两个说话者在未来约 0–0.6 秒的相对活动预测。
|
||||
p_future：两个说话者在未来约 0.6–2 秒的相对活动预测。
|
||||
vad：两个说话者当前语音活动概率。
|
||||
这三个输出均按 [用户, 机器人] 排列，不能直接当作 can_reply 或 EOT 概率。
|
||||
MaAI 的原始 t 是墙钟时间；脚本另记 audio_end_sec 作为录音媒体时间。
|
||||
|
||||
兼容处理
|
||||
使用本地官方预训练权重，避免自动下载或写入全局模型缓存。
|
||||
旧 CPC 权重含 argparse.Namespace，脚本仅在构造模型时通过 safe_globals 允许该类型，保持 PyTorch weights_only 安全加载。
|
||||
依赖恢复可用 uv sync --locked。新机器需先 brew install portaudio；构建 PyAudio 必要时添加：
|
||||
CFLAGS="-I/opt/homebrew/opt/portaudio/include" LDFLAGS="-L/opt/homebrew/opt/portaudio/lib" uv sync --locked
|
||||
Reference in New Issue
Block a user