280 lines
8.2 KiB
JSON
280 lines
8.2 KiB
JSON
{
|
|
"model": "VAP / MaAI (user only, silent robot)",
|
|
"total_points": 44,
|
|
"created_at": "2026-10-27T04:12:35.871425+00:00",
|
|
"main_rule": "1 - p_now[user] > 0.5",
|
|
"main_metrics": {
|
|
"n": 44,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.5135135135135135,
|
|
"f1": 0.6440677966101694
|
|
},
|
|
"thresholds": [
|
|
{
|
|
"threshold": 0.11,
|
|
"n": 44,
|
|
"tn": 2,
|
|
"fp": 20,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.9090909090909091,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5238095238095238,
|
|
"f1": 0.6875
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.5135135135135135,
|
|
"f1": 0.6440677966101694
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"tn": 10,
|
|
"fp": 12,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.6590909090909091,
|
|
"false_reply_rate": 0.5454545454545454,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.6129032258064516,
|
|
"f1": 0.7169811320754716
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"tn": 13,
|
|
"fp": 9,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.7272727272727273,
|
|
"false_reply_rate": 0.4090909090909091,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.6785714285714286,
|
|
"f1": 0.76
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"tn": 20,
|
|
"fp": 2,
|
|
"fn": 9,
|
|
"tp": 13,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.09090909090909091,
|
|
"false_hold_rate": 0.4090909090909091,
|
|
"recall": 0.5909090909090909,
|
|
"precision": 0.8666666666666667,
|
|
"f1": 0.7027027027027027
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546,
|
|
"recall": 0.045454545454545456,
|
|
"precision": 1.0,
|
|
"f1": 0.08695652173913043
|
|
},
|
|
{
|
|
"threshold": 0.98,
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
}
|
|
],
|
|
"same_point_stereo_reference": {
|
|
"n": 44,
|
|
"tn": 14,
|
|
"fp": 8,
|
|
"fn": 4,
|
|
"tp": 18,
|
|
"accuracy": 0.7272727272727273,
|
|
"false_reply_rate": 0.36363636363636365,
|
|
"false_hold_rate": 0.18181818181818182,
|
|
"recall": 0.8181818181818182,
|
|
"precision": 0.6923076923076923,
|
|
"f1": 0.75
|
|
},
|
|
"changed_vs_stereo": {
|
|
"fixed": 4,
|
|
"regressed": 13
|
|
},
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"n": 6,
|
|
"tn": 0,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.6,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"n": 8,
|
|
"tn": 1,
|
|
"fp": 3,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 0.625,
|
|
"false_reply_rate": 0.75,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5714285714285714,
|
|
"f1": 0.7272727272727273
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"n": 9,
|
|
"tn": 1,
|
|
"fp": 3,
|
|
"fn": 2,
|
|
"tp": 3,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 0.75,
|
|
"false_hold_rate": 0.4,
|
|
"recall": 0.6,
|
|
"precision": 0.5,
|
|
"f1": 0.5454545454545454
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"n": 5,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"n": 9,
|
|
"tn": 1,
|
|
"fp": 5,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 0.8333333333333334,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.375,
|
|
"f1": 0.5454545454545454
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"n": 7,
|
|
"tn": 0,
|
|
"fp": 3,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 0.5714285714285714,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5714285714285714,
|
|
"f1": 0.7272727272727273
|
|
}
|
|
},
|
|
"annotation_sources": {
|
|
"xlsx_draft": 29,
|
|
"unspecified": 15
|
|
},
|
|
"skipped_unlabeled": 0,
|
|
"witness": {
|
|
"seed": 42,
|
|
"robot_and_future_invariance": true,
|
|
"repeat_identical": true,
|
|
"score": 0.9077615067362785
|
|
},
|
|
"provenance": {
|
|
"machine": "macOS-26.4.1-arm64-arm-64bit",
|
|
"hardware": "Apple M4 Mac mini / 16 GB",
|
|
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
|
|
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
|
|
"device": "cpu",
|
|
"threads": 4,
|
|
"dependencies": {
|
|
"maai": "0.2.22",
|
|
"torch": "2.14.1",
|
|
"transformers": "5.5.3",
|
|
"numpy": "2.4.6",
|
|
"soundfile": "0.14.0",
|
|
"soxr": "1.1.0"
|
|
},
|
|
"sample_rate": 16000,
|
|
"frame_ms": 100,
|
|
"context_ms": 20000,
|
|
"use_kv_cache": true,
|
|
"seed": 42,
|
|
"channel_mapping": {
|
|
"0": "original user",
|
|
"1": "all-zero robot input"
|
|
},
|
|
"weight_sha256": {
|
|
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
|
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
|
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
|
|
"script_sha256": "93c8a69850cd444c8daa9525eda267cda5f4ca34393df5f175286d875aa19195",
|
|
"model_factory_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe",
|
|
"loader_sha256": "4a7b37e6b67e5c55c030bcc399ab58e6575e49592dbda87dc311ec88da38e1d2",
|
|
"audio_sha256": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
|
}
|
|
},
|
|
"method": "At each exact point, independently reset CPC/VAP and replay <=20s in 100ms frames. User channel unchanged; robot channel zero for the full context before resampling. Also rerun the stereo control on identical current points. No annotation transcripts, future audio, end signal, silence gate, or timeout. Score is an activity-projection proxy, not calibrated EOT probability.",
|
|
"limitations": [
|
|
"Exploratory selected-point evaluation, not a held-out benchmark or endpointing-policy evaluation.",
|
|
"Continuous full-call KV/CPC state differs from independent point replay; curve and checkpoint scores may differ.",
|
|
"Zeroing the robot channel cannot remove robot leakage already present in the user channel.",
|
|
"Stereo reference here is rerun at current annotation points; existing legacy frontend checkpoints are preserved separately."
|
|
],
|
|
"timeline_points": 4873,
|
|
"elapsed_seconds": 124.26453408319503
|
|
}
|