280 lines
8.2 KiB
JSON
280 lines
8.2 KiB
JSON
|
{
|
||
|
|
"model": "VAP / MaAI (user only, silent robot)",
|
||
|
|
"total_points": 44,
|
||
|
|
"created_at": "2026-10-27T04:12:35.871425+00:00",
|
||
|
|
"main_rule": "1 - p_now[user] > 0.5",
|
||
|
|
"main_metrics": {
|
||
|
|
"n": 44,
|
||
|
|
"tn": 4,
|
||
|
|
"fp": 18,
|
||
|
|
"fn": 3,
|
||
|
|
"tp": 19,
|
||
|
|
"accuracy": 0.5227272727272727,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.13636363636363635,
|
||
|
|
"recall": 0.8636363636363636,
|
||
|
|
"precision": 0.5135135135135135,
|
||
|
|
"f1": 0.6440677966101694
|
||
|
|
},
|
||
|
|
"thresholds": [
|
||
|
|
{
|
||
|
|
"threshold": 0.11,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 20,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 22,
|
||
|
|
"accuracy": 0.5454545454545454,
|
||
|
|
"false_reply_rate": 0.9090909090909091,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5238095238095238,
|
||
|
|
"f1": 0.6875
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.5,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 4,
|
||
|
|
"fp": 18,
|
||
|
|
"fn": 3,
|
||
|
|
"tp": 19,
|
||
|
|
"accuracy": 0.5227272727272727,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.13636363636363635,
|
||
|
|
"recall": 0.8636363636363636,
|
||
|
|
"precision": 0.5135135135135135,
|
||
|
|
"f1": 0.6440677966101694
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.7,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 10,
|
||
|
|
"fp": 12,
|
||
|
|
"fn": 3,
|
||
|
|
"tp": 19,
|
||
|
|
"accuracy": 0.6590909090909091,
|
||
|
|
"false_reply_rate": 0.5454545454545454,
|
||
|
|
"false_hold_rate": 0.13636363636363635,
|
||
|
|
"recall": 0.8636363636363636,
|
||
|
|
"precision": 0.6129032258064516,
|
||
|
|
"f1": 0.7169811320754716
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.8,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 13,
|
||
|
|
"fp": 9,
|
||
|
|
"fn": 3,
|
||
|
|
"tp": 19,
|
||
|
|
"accuracy": 0.7272727272727273,
|
||
|
|
"false_reply_rate": 0.4090909090909091,
|
||
|
|
"false_hold_rate": 0.13636363636363635,
|
||
|
|
"recall": 0.8636363636363636,
|
||
|
|
"precision": 0.6785714285714286,
|
||
|
|
"f1": 0.76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.9,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 20,
|
||
|
|
"fp": 2,
|
||
|
|
"fn": 9,
|
||
|
|
"tp": 13,
|
||
|
|
"accuracy": 0.75,
|
||
|
|
"false_reply_rate": 0.09090909090909091,
|
||
|
|
"false_hold_rate": 0.4090909090909091,
|
||
|
|
"recall": 0.5909090909090909,
|
||
|
|
"precision": 0.8666666666666667,
|
||
|
|
"f1": 0.7027027027027027
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.95,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 22,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 21,
|
||
|
|
"tp": 1,
|
||
|
|
"accuracy": 0.5227272727272727,
|
||
|
|
"false_reply_rate": 0.0,
|
||
|
|
"false_hold_rate": 0.9545454545454546,
|
||
|
|
"recall": 0.045454545454545456,
|
||
|
|
"precision": 1.0,
|
||
|
|
"f1": 0.08695652173913043
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.98,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 22,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 22,
|
||
|
|
"tp": 0,
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"false_reply_rate": 0.0,
|
||
|
|
"false_hold_rate": 1.0,
|
||
|
|
"recall": 0.0,
|
||
|
|
"precision": null,
|
||
|
|
"f1": 0.0
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"same_point_stereo_reference": {
|
||
|
|
"n": 44,
|
||
|
|
"tn": 14,
|
||
|
|
"fp": 8,
|
||
|
|
"fn": 4,
|
||
|
|
"tp": 18,
|
||
|
|
"accuracy": 0.7272727272727273,
|
||
|
|
"false_reply_rate": 0.36363636363636365,
|
||
|
|
"false_hold_rate": 0.18181818181818182,
|
||
|
|
"recall": 0.8181818181818182,
|
||
|
|
"precision": 0.6923076923076923,
|
||
|
|
"f1": 0.75
|
||
|
|
},
|
||
|
|
"changed_vs_stereo": {
|
||
|
|
"fixed": 4,
|
||
|
|
"regressed": 13
|
||
|
|
},
|
||
|
|
"per_call": {
|
||
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||
|
|
"n": 6,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 2,
|
||
|
|
"fn": 1,
|
||
|
|
"tp": 3,
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.25,
|
||
|
|
"recall": 0.75,
|
||
|
|
"precision": 0.6,
|
||
|
|
"f1": 0.6666666666666666
|
||
|
|
},
|
||
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||
|
|
"n": 8,
|
||
|
|
"tn": 1,
|
||
|
|
"fp": 3,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.625,
|
||
|
|
"false_reply_rate": 0.75,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5714285714285714,
|
||
|
|
"f1": 0.7272727272727273
|
||
|
|
},
|
||
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||
|
|
"n": 9,
|
||
|
|
"tn": 1,
|
||
|
|
"fp": 3,
|
||
|
|
"fn": 2,
|
||
|
|
"tp": 3,
|
||
|
|
"accuracy": 0.4444444444444444,
|
||
|
|
"false_reply_rate": 0.75,
|
||
|
|
"false_hold_rate": 0.4,
|
||
|
|
"recall": 0.6,
|
||
|
|
"precision": 0.5,
|
||
|
|
"f1": 0.5454545454545454
|
||
|
|
},
|
||
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||
|
|
"n": 5,
|
||
|
|
"tn": 1,
|
||
|
|
"fp": 2,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 2,
|
||
|
|
"accuracy": 0.6,
|
||
|
|
"false_reply_rate": 0.6666666666666666,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5,
|
||
|
|
"f1": 0.6666666666666666
|
||
|
|
},
|
||
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||
|
|
"n": 9,
|
||
|
|
"tn": 1,
|
||
|
|
"fp": 5,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 3,
|
||
|
|
"accuracy": 0.4444444444444444,
|
||
|
|
"false_reply_rate": 0.8333333333333334,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.375,
|
||
|
|
"f1": 0.5454545454545454
|
||
|
|
},
|
||
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||
|
|
"n": 7,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 3,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.5714285714285714,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5714285714285714,
|
||
|
|
"f1": 0.7272727272727273
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"annotation_sources": {
|
||
|
|
"xlsx_draft": 29,
|
||
|
|
"unspecified": 15
|
||
|
|
},
|
||
|
|
"skipped_unlabeled": 0,
|
||
|
|
"witness": {
|
||
|
|
"seed": 42,
|
||
|
|
"robot_and_future_invariance": true,
|
||
|
|
"repeat_identical": true,
|
||
|
|
"score": 0.9077615067362785
|
||
|
|
},
|
||
|
|
"provenance": {
|
||
|
|
"machine": "macOS-26.4.1-arm64-arm-64bit",
|
||
|
|
"hardware": "Apple M4 Mac mini / 16 GB",
|
||
|
|
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
|
||
|
|
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
|
||
|
|
"device": "cpu",
|
||
|
|
"threads": 4,
|
||
|
|
"dependencies": {
|
||
|
|
"maai": "0.2.22",
|
||
|
|
"torch": "2.14.1",
|
||
|
|
"transformers": "5.5.3",
|
||
|
|
"numpy": "2.4.6",
|
||
|
|
"soundfile": "0.14.0",
|
||
|
|
"soxr": "1.1.0"
|
||
|
|
},
|
||
|
|
"sample_rate": 16000,
|
||
|
|
"frame_ms": 100,
|
||
|
|
"context_ms": 20000,
|
||
|
|
"use_kv_cache": true,
|
||
|
|
"seed": 42,
|
||
|
|
"channel_mapping": {
|
||
|
|
"0": "original user",
|
||
|
|
"1": "all-zero robot input"
|
||
|
|
},
|
||
|
|
"weight_sha256": {
|
||
|
|
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
||
|
|
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
||
|
|
},
|
||
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
||
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
||
|
|
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
|
||
|
|
"script_sha256": "93c8a69850cd444c8daa9525eda267cda5f4ca34393df5f175286d875aa19195",
|
||
|
|
"model_factory_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe",
|
||
|
|
"loader_sha256": "4a7b37e6b67e5c55c030bcc399ab58e6575e49592dbda87dc311ec88da38e1d2",
|
||
|
|
"audio_sha256": {
|
||
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
||
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
||
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
||
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
||
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
||
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"method": "At each exact point, independently reset CPC/VAP and replay <=20s in 100ms frames. User channel unchanged; robot channel zero for the full context before resampling. Also rerun the stereo control on identical current points. No annotation transcripts, future audio, end signal, silence gate, or timeout. Score is an activity-projection proxy, not calibrated EOT probability.",
|
||
|
|
"limitations": [
|
||
|
|
"Exploratory selected-point evaluation, not a held-out benchmark or endpointing-policy evaluation.",
|
||
|
|
"Continuous full-call KV/CPC state differs from independent point replay; curve and checkpoint scores may differ.",
|
||
|
|
"Zeroing the robot channel cannot remove robot leakage already present in the user channel.",
|
||
|
|
"Stereo reference here is rerun at current annotation points; existing legacy frontend checkpoints are preserved separately."
|
||
|
|
],
|
||
|
|
"timeline_points": 4873,
|
||
|
|
"elapsed_seconds": 124.26453408319503
|
||
|
|
}
|