wx44wx/xiaobailing-turn-detection
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增 Mimi 与仅用户 VAP 对比并统一流式评测

Browse Source
This commit is contained in:
Xin Wang
2026-10-28 22:44:36 +08:00
parent 19d4e3c767
commit 6785e3a07b
55 changed files with 6160 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/maai-ch-kyoto/results/user-only/summary.json
+279
View File
@@ -0,0 +1,279 @@
{
"model": "VAP / MaAI (user only, silent robot)",
"total_points": 44,
"created_at": "2026-10-27T04:12:35.871425+00:00",
"main_rule": "1 - p_now[user] > 0.5",
"main_metrics": {
"n": 44,
"tn": 4,
"fp": 18,
"fn": 3,
"tp": 19,
"accuracy": 0.5227272727272727,
"false_reply_rate": 0.8181818181818182,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.5135135135135135,
"f1": 0.6440677966101694
},
"thresholds": [
{
"threshold": 0.11,
"n": 44,
"tn": 2,
"fp": 20,
"fn": 0,
"tp": 22,
"accuracy": 0.5454545454545454,
"false_reply_rate": 0.9090909090909091,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.5238095238095238,
"f1": 0.6875
},
{
"threshold": 0.5,
"n": 44,
"tn": 4,
"fp": 18,
"fn": 3,
"tp": 19,
"accuracy": 0.5227272727272727,
"false_reply_rate": 0.8181818181818182,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.5135135135135135,
"f1": 0.6440677966101694
},
{
"threshold": 0.7,
"n": 44,
"tn": 10,
"fp": 12,
"fn": 3,
"tp": 19,
"accuracy": 0.6590909090909091,
"false_reply_rate": 0.5454545454545454,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.6129032258064516,
"f1": 0.7169811320754716
},
{
"threshold": 0.8,
"n": 44,
"tn": 13,
"fp": 9,
"fn": 3,
"tp": 19,
"accuracy": 0.7272727272727273,
"false_reply_rate": 0.4090909090909091,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.6785714285714286,
"f1": 0.76
},
{
"threshold": 0.9,
"n": 44,
"tn": 20,
"fp": 2,
"fn": 9,
"tp": 13,
"accuracy": 0.75,
"false_reply_rate": 0.09090909090909091,
"false_hold_rate": 0.4090909090909091,
"recall": 0.5909090909090909,
"precision": 0.8666666666666667,
"f1": 0.7027027027027027
},
{
"threshold": 0.95,
"n": 44,
"tn": 22,
"fp": 0,
"fn": 21,
"tp": 1,
"accuracy": 0.5227272727272727,
"false_reply_rate": 0.0,
"false_hold_rate": 0.9545454545454546,
"recall": 0.045454545454545456,
"precision": 1.0,
"f1": 0.08695652173913043
},
{
"threshold": 0.98,
"n": 44,
"tn": 22,
"fp": 0,
"fn": 22,
"tp": 0,
"accuracy": 0.5,
"false_reply_rate": 0.0,
"false_hold_rate": 1.0,
"recall": 0.0,
"precision": null,
"f1": 0.0
}
],
"same_point_stereo_reference": {
"n": 44,
"tn": 14,
"fp": 8,
"fn": 4,
"tp": 18,
"accuracy": 0.7272727272727273,
"false_reply_rate": 0.36363636363636365,
"false_hold_rate": 0.18181818181818182,
"recall": 0.8181818181818182,
"precision": 0.6923076923076923,
"f1": 0.75
},
"changed_vs_stereo": {
"fixed": 4,
"regressed": 13
},
"per_call": {
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
"n": 6,
"tn": 0,
"fp": 2,
"fn": 1,
"tp": 3,
"accuracy": 0.5,
"false_reply_rate": 1.0,
"false_hold_rate": 0.25,
"recall": 0.75,
"precision": 0.6,
"f1": 0.6666666666666666
},
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
"n": 8,
"tn": 1,
"fp": 3,
"fn": 0,
"tp": 4,
"accuracy": 0.625,
"false_reply_rate": 0.75,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.5714285714285714,
"f1": 0.7272727272727273
},
"872a1e67-873a-4c49-a598-b38a670afcad": {
"n": 9,
"tn": 1,
"fp": 3,
"fn": 2,
"tp": 3,
"accuracy": 0.4444444444444444,
"false_reply_rate": 0.75,
"false_hold_rate": 0.4,
"recall": 0.6,
"precision": 0.5,
"f1": 0.5454545454545454
},
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
"n": 5,
"tn": 1,
"fp": 2,
"fn": 0,
"tp": 2,
"accuracy": 0.6,
"false_reply_rate": 0.6666666666666666,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.5,
"f1": 0.6666666666666666
},
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
"n": 9,
"tn": 1,
"fp": 5,
"fn": 0,
"tp": 3,
"accuracy": 0.4444444444444444,
"false_reply_rate": 0.8333333333333334,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.375,
"f1": 0.5454545454545454
},
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
"n": 7,
"tn": 0,
"fp": 3,
"fn": 0,
"tp": 4,
"accuracy": 0.5714285714285714,
"false_reply_rate": 1.0,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.5714285714285714,
"f1": 0.7272727272727273
}
},
"annotation_sources": {
"xlsx_draft": 29,
"unspecified": 15
},
"skipped_unlabeled": 0,
"witness": {
"seed": 42,
"robot_and_future_invariance": true,
"repeat_identical": true,
"score": 0.9077615067362785
},
"provenance": {
"machine": "macOS-26.4.1-arm64-arm-64bit",
"hardware": "Apple M4 Mac mini / 16 GB",
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
"device": "cpu",
"threads": 4,
"dependencies": {
"maai": "0.2.22",
"torch": "2.14.1",
"transformers": "5.5.3",
"numpy": "2.4.6",
"soundfile": "0.14.0",
"soxr": "1.1.0"
},
"sample_rate": 16000,
"frame_ms": 100,
"context_ms": 20000,
"use_kv_cache": true,
"seed": 42,
"channel_mapping": {
"0": "original user",
"1": "all-zero robot input"
},
"weight_sha256": {
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
},
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
"script_sha256": "93c8a69850cd444c8daa9525eda267cda5f4ca34393df5f175286d875aa19195",
"model_factory_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe",
"loader_sha256": "4a7b37e6b67e5c55c030bcc399ab58e6575e49592dbda87dc311ec88da38e1d2",
"audio_sha256": {
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
}
},
"method": "At each exact point, independently reset CPC/VAP and replay <=20s in 100ms frames. User channel unchanged; robot channel zero for the full context before resampling. Also rerun the stereo control on identical current points. No annotation transcripts, future audio, end signal, silence gate, or timeout. Score is an activity-projection proxy, not calibrated EOT probability.",
"limitations": [
"Exploratory selected-point evaluation, not a held-out benchmark or endpointing-policy evaluation.",
"Continuous full-call KV/CPC state differs from independent point replay; curve and checkpoint scores may differ.",
"Zeroing the robot channel cannot remove robot leakage already present in the user channel.",
"Stereo reference here is rerun at current annotation points; existing legacy frontend checkpoints are preserved separately."
],
"timeline_points": 4873,
"elapsed_seconds": 124.26453408319503
}