378 lines
9.6 KiB
JSON
378 lines
9.6 KiB
JSON
{
|
|
"threshold": 0.5,
|
|
"rule": "1 - p_now[user] > 0.5",
|
|
"n": 44,
|
|
"positive": 22,
|
|
"models": {
|
|
"cpc_mc_stereo": {
|
|
"n": 44,
|
|
"correct": 32,
|
|
"tn": 14,
|
|
"fp": 8,
|
|
"fn": 4,
|
|
"tp": 18,
|
|
"accuracy": 0.7272727272727273,
|
|
"false_reply_rate": 0.36363636363636365,
|
|
"false_hold_rate": 0.18181818181818182,
|
|
"accuracy_delta_pp_vs_cpc_mc_stereo": 0.0,
|
|
"relative_accuracy_change_vs_cpc_mc_stereo": 0.0
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 44,
|
|
"correct": 23,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"accuracy_delta_pp_vs_cpc_mc_stereo": -20.45454545454546,
|
|
"relative_accuracy_change_vs_cpc_mc_stereo": -0.28125000000000006
|
|
},
|
|
"mimi_standard": {
|
|
"n": 44,
|
|
"correct": 30,
|
|
"tn": 15,
|
|
"fp": 7,
|
|
"fn": 7,
|
|
"tp": 15,
|
|
"accuracy": 0.6818181818181818,
|
|
"false_reply_rate": 0.3181818181818182,
|
|
"false_hold_rate": 0.3181818181818182,
|
|
"accuracy_delta_pp_vs_cpc_mc_stereo": -4.545454545454552,
|
|
"relative_accuracy_change_vs_cpc_mc_stereo": -0.0625000000000001
|
|
},
|
|
"mimi_mc": {
|
|
"n": 44,
|
|
"correct": 34,
|
|
"tn": 15,
|
|
"fp": 7,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.7727272727272727,
|
|
"false_reply_rate": 0.3181818181818182,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"accuracy_delta_pp_vs_cpc_mc_stereo": 4.545454545454541,
|
|
"relative_accuracy_change_vs_cpc_mc_stereo": 0.062499999999999944
|
|
}
|
|
},
|
|
"paired_vs_cpc_mc_stereo": {
|
|
"mimi_standard": {
|
|
"fixed": 4,
|
|
"regressed": 6
|
|
},
|
|
"mimi_mc": {
|
|
"fixed": 5,
|
|
"regressed": 3
|
|
}
|
|
},
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"cpc_mc_stereo": {
|
|
"n": 6,
|
|
"correct": 3,
|
|
"tn": 0,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.25
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 6,
|
|
"correct": 3,
|
|
"tn": 0,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.25
|
|
},
|
|
"mimi_standard": {
|
|
"n": 6,
|
|
"correct": 4,
|
|
"tn": 1,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.6666666666666666,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.25
|
|
},
|
|
"mimi_mc": {
|
|
"n": 6,
|
|
"correct": 4,
|
|
"tn": 1,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.6666666666666666,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.25
|
|
}
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"cpc_mc_stereo": {
|
|
"n": 8,
|
|
"correct": 5,
|
|
"tn": 2,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.625,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.25
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 8,
|
|
"correct": 5,
|
|
"tn": 1,
|
|
"fp": 3,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 0.625,
|
|
"false_reply_rate": 0.75,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_standard": {
|
|
"n": 8,
|
|
"correct": 5,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 2,
|
|
"tp": 2,
|
|
"accuracy": 0.625,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.5
|
|
},
|
|
"mimi_mc": {
|
|
"n": 8,
|
|
"correct": 6,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.25
|
|
}
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"cpc_mc_stereo": {
|
|
"n": 9,
|
|
"correct": 7,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 4,
|
|
"accuracy": 0.7777777777777778,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.2
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 9,
|
|
"correct": 4,
|
|
"tn": 1,
|
|
"fp": 3,
|
|
"fn": 2,
|
|
"tp": 3,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 0.75,
|
|
"false_hold_rate": 0.4
|
|
},
|
|
"mimi_standard": {
|
|
"n": 9,
|
|
"correct": 5,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 3,
|
|
"tp": 2,
|
|
"accuracy": 0.5555555555555556,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.6
|
|
},
|
|
"mimi_mc": {
|
|
"n": 9,
|
|
"correct": 7,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 4,
|
|
"accuracy": 0.7777777777777778,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.2
|
|
}
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"cpc_mc_stereo": {
|
|
"n": 5,
|
|
"correct": 3,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 5,
|
|
"correct": 3,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_standard": {
|
|
"n": 5,
|
|
"correct": 2,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 1,
|
|
"accuracy": 0.4,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.5
|
|
},
|
|
"mimi_mc": {
|
|
"n": 5,
|
|
"correct": 3,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0
|
|
}
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"cpc_mc_stereo": {
|
|
"n": 9,
|
|
"correct": 8,
|
|
"tn": 5,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"accuracy": 0.8888888888888888,
|
|
"false_reply_rate": 0.16666666666666666,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 9,
|
|
"correct": 4,
|
|
"tn": 1,
|
|
"fp": 5,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 0.8333333333333334,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_standard": {
|
|
"n": 9,
|
|
"correct": 7,
|
|
"tn": 4,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"accuracy": 0.7777777777777778,
|
|
"false_reply_rate": 0.3333333333333333,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_mc": {
|
|
"n": 9,
|
|
"correct": 7,
|
|
"tn": 4,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"accuracy": 0.7777777777777778,
|
|
"false_reply_rate": 0.3333333333333333,
|
|
"false_hold_rate": 0.0
|
|
}
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"cpc_mc_stereo": {
|
|
"n": 7,
|
|
"correct": 6,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"accuracy": 0.8571428571428571,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.25
|
|
},
|
|
"cpc_mc_user_only": {
|
|
"n": 7,
|
|
"correct": 4,
|
|
"tn": 0,
|
|
"fp": 3,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 0.5714285714285714,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_standard": {
|
|
"n": 7,
|
|
"correct": 7,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 1.0,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"mimi_mc": {
|
|
"n": 7,
|
|
"correct": 7,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"accuracy": 1.0,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.0
|
|
}
|
|
}
|
|
},
|
|
"sources": {
|
|
"mimi-standard": {
|
|
"summary_sha256": "b2524af78cd9768a6ee4f3c73e48a6bfc8f5482274acb503bdb0881f70a2ab29",
|
|
"predictions_sha256": "1f10ead96648b26ec5bea0cee0ed5dfd7c357886b162a44e8b46f4648f4dfcf8"
|
|
},
|
|
"mimi-mc": {
|
|
"summary_sha256": "f7f87c19dfd894771896ce798683b7d2f18bcf0bc2e13da185b9f6c71646359f",
|
|
"predictions_sha256": "80d850b0044e365cf4554e116f3617af72c7e4a56eb44aaede00004856c20a2b"
|
|
},
|
|
"user-only": {
|
|
"summary_sha256": "99577db77517e31663256c2489ff2300883485ff85a1d04608e467a0bf7e6f9a",
|
|
"predictions_sha256": "266f43c91d232e5fa7e8e9fc10c294c8f7e7df64fac2332a6a7441faf15d30a8"
|
|
}
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"audio_sha256": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
|
},
|
|
"limitations": [
|
|
"Single seed 42 with exact-repeat witness, not multiple-seed statistics.",
|
|
"Only 44 selected points in six calls; descriptive comparison, not a held-out generalization claim.",
|
|
"Mimi standard/MC share the same encoder and framing; CPC uses a different encoder and native 100ms framing.",
|
|
"All scores are activity-projection mappings; no endpointing-policy or full-dialogue cutoff/latency claims."
|
|
]
|
|
}
|