1272 lines
45 KiB
JSON
1272 lines
45 KiB
JSON
{
|
|
"evaluation_mode": "streaming",
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"models": {
|
|
"maai": {
|
|
"model": "maai",
|
|
"name": "CPC 降噪双声道",
|
|
"evaluation_mode": "streaming",
|
|
"created_at": "2026-10-28T00:27:46.310011+00:00",
|
|
"threshold": 0.5,
|
|
"rule": "1 - p_now[user] > 0.5",
|
|
"main_metrics": {
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 15,
|
|
"fp": 7,
|
|
"fn": 7,
|
|
"tp": 15,
|
|
"correct": 30,
|
|
"accuracy": 0.6818181818181818,
|
|
"false_reply_rate": 0.3181818181818182,
|
|
"false_hold_rate": 0.3181818181818182,
|
|
"recall": 0.6818181818181818,
|
|
"precision": 0.6818181818181818,
|
|
"f1": 0.6818181818181818
|
|
},
|
|
"thresholds": [
|
|
{
|
|
"threshold": 0.11,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 7,
|
|
"fp": 15,
|
|
"fn": 1,
|
|
"tp": 21,
|
|
"correct": 28,
|
|
"accuracy": 0.6363636363636364,
|
|
"false_reply_rate": 0.6818181818181818,
|
|
"false_hold_rate": 0.045454545454545456,
|
|
"recall": 0.9545454545454546,
|
|
"precision": 0.5833333333333334,
|
|
"f1": 0.7241379310344828
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 15,
|
|
"fp": 7,
|
|
"fn": 7,
|
|
"tp": 15,
|
|
"correct": 30,
|
|
"accuracy": 0.6818181818181818,
|
|
"false_reply_rate": 0.3181818181818182,
|
|
"false_hold_rate": 0.3181818181818182,
|
|
"recall": 0.6818181818181818,
|
|
"precision": 0.6818181818181818,
|
|
"f1": 0.6818181818181818
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 18,
|
|
"fp": 4,
|
|
"fn": 11,
|
|
"tp": 11,
|
|
"correct": 29,
|
|
"accuracy": 0.6590909090909091,
|
|
"false_reply_rate": 0.18181818181818182,
|
|
"false_hold_rate": 0.5,
|
|
"recall": 0.5,
|
|
"precision": 0.7333333333333333,
|
|
"f1": 0.5945945945945946
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 19,
|
|
"fp": 3,
|
|
"fn": 15,
|
|
"tp": 7,
|
|
"correct": 26,
|
|
"accuracy": 0.5909090909090909,
|
|
"false_reply_rate": 0.13636363636363635,
|
|
"false_hold_rate": 0.6818181818181818,
|
|
"recall": 0.3181818181818182,
|
|
"precision": 0.7,
|
|
"f1": 0.4375
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 19,
|
|
"tp": 3,
|
|
"correct": 24,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.8636363636363636,
|
|
"recall": 0.13636363636363635,
|
|
"precision": 0.75,
|
|
"f1": 0.23076923076923078
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"correct": 23,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546,
|
|
"recall": 0.045454545454545456,
|
|
"precision": 1.0,
|
|
"f1": 0.08695652173913043
|
|
},
|
|
{
|
|
"threshold": 0.98,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"correct": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
}
|
|
],
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"n": 6,
|
|
"requested_labeled": 6,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 4,
|
|
"accuracy": 0.6666666666666666,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.75,
|
|
"f1": 0.75
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"n": 8,
|
|
"requested_labeled": 8,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.75,
|
|
"f1": 0.75
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 2,
|
|
"fp": 2,
|
|
"fn": 2,
|
|
"tp": 3,
|
|
"correct": 5,
|
|
"accuracy": 0.5555555555555556,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.4,
|
|
"recall": 0.6,
|
|
"precision": 0.6,
|
|
"f1": 0.6
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"n": 5,
|
|
"requested_labeled": 5,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 1,
|
|
"correct": 2,
|
|
"accuracy": 0.4,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.5,
|
|
"recall": 0.5,
|
|
"precision": 0.3333333333333333,
|
|
"f1": 0.4
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 5,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"correct": 8,
|
|
"accuracy": 0.8888888888888888,
|
|
"false_reply_rate": 0.16666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.75,
|
|
"f1": 0.8571428571428571
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"n": 7,
|
|
"requested_labeled": 7,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 2,
|
|
"tp": 2,
|
|
"correct": 5,
|
|
"accuracy": 0.7142857142857143,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.5,
|
|
"recall": 0.5,
|
|
"precision": 1.0,
|
|
"f1": 0.6666666666666666
|
|
}
|
|
},
|
|
"method": "Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. No state reset, future score, interpolation or independent-window checkpoint. Availability includes source resampler buffering; score is the stored activity-projection proxy.",
|
|
"provenance": {
|
|
"source_export": "experiments/maai-ch-kyoto/results/score-timeline.json",
|
|
"source_export_sha256": "492db23873849516eec7449038a861cec2969dc0cc9de4877002d945fcb42d24",
|
|
"source_inference_provenance": {},
|
|
"source_model_sha256": {
|
|
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
|
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"evaluation_script_sha256": "85d0b5f0f8dcf73cc6ee1b651e5a2cfc0b40e95024aacb0bd3ed975079ad9507"
|
|
},
|
|
"limitations": [
|
|
"Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.",
|
|
"Uses stored rounded continuous scores; no new inference or model-speed measurement.",
|
|
"No prior output or stale output is unavailable, not a negative/zero prediction.",
|
|
"Archived independent-window results are a separate evaluation protocol."
|
|
],
|
|
"accuracy_delta_pp_vs_cpc_stereo": 0.0,
|
|
"relative_accuracy_change_vs_cpc_stereo": 0.0
|
|
},
|
|
"maai_user_only": {
|
|
"model": "maai_user_only",
|
|
"name": "CPC 降噪机器人静音",
|
|
"evaluation_mode": "streaming",
|
|
"created_at": "2026-10-28T00:27:46.321764+00:00",
|
|
"threshold": 0.5,
|
|
"rule": "1 - p_now[user] > 0.5",
|
|
"main_metrics": {
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 2,
|
|
"tp": 20,
|
|
"correct": 24,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.09090909090909091,
|
|
"recall": 0.9090909090909091,
|
|
"precision": 0.5263157894736842,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"thresholds": [
|
|
{
|
|
"threshold": 0.11,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 19,
|
|
"fn": 1,
|
|
"tp": 21,
|
|
"correct": 24,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.8636363636363636,
|
|
"false_hold_rate": 0.045454545454545456,
|
|
"recall": 0.9545454545454546,
|
|
"precision": 0.525,
|
|
"f1": 0.6774193548387096
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 2,
|
|
"tp": 20,
|
|
"correct": 24,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.09090909090909091,
|
|
"recall": 0.9090909090909091,
|
|
"precision": 0.5263157894736842,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 11,
|
|
"fp": 11,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"correct": 30,
|
|
"accuracy": 0.6818181818181818,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.6333333333333333,
|
|
"f1": 0.7307692307692307
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 16,
|
|
"fp": 6,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"correct": 35,
|
|
"accuracy": 0.7954545454545454,
|
|
"false_reply_rate": 0.2727272727272727,
|
|
"false_hold_rate": 0.13636363636363635,
|
|
"recall": 0.8636363636363636,
|
|
"precision": 0.76,
|
|
"f1": 0.8085106382978723
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 12,
|
|
"tp": 10,
|
|
"correct": 31,
|
|
"accuracy": 0.7045454545454546,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.5454545454545454,
|
|
"recall": 0.45454545454545453,
|
|
"precision": 0.9090909090909091,
|
|
"f1": 0.6060606060606061
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"correct": 23,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546,
|
|
"recall": 0.045454545454545456,
|
|
"precision": 1.0,
|
|
"f1": 0.08695652173913043
|
|
},
|
|
{
|
|
"threshold": 0.98,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"correct": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
}
|
|
],
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"n": 6,
|
|
"requested_labeled": 6,
|
|
"unavailable_labeled": 0,
|
|
"tn": 0,
|
|
"fp": 2,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 3,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.6,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"n": 8,
|
|
"requested_labeled": 8,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 3,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"correct": 5,
|
|
"accuracy": 0.625,
|
|
"false_reply_rate": 0.75,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5714285714285714,
|
|
"f1": 0.7272727272727273
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 0,
|
|
"fp": 4,
|
|
"fn": 1,
|
|
"tp": 4,
|
|
"correct": 4,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.2,
|
|
"recall": 0.8,
|
|
"precision": 0.5,
|
|
"f1": 0.6153846153846154
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"n": 5,
|
|
"requested_labeled": 5,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"correct": 3,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.5,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 5,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"correct": 4,
|
|
"accuracy": 0.4444444444444444,
|
|
"false_reply_rate": 0.8333333333333334,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.375,
|
|
"f1": 0.5454545454545454
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"n": 7,
|
|
"requested_labeled": 7,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 2,
|
|
"fn": 0,
|
|
"tp": 4,
|
|
"correct": 5,
|
|
"accuracy": 0.7142857142857143,
|
|
"false_reply_rate": 0.6666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.6666666666666666,
|
|
"f1": 0.8
|
|
}
|
|
},
|
|
"method": "Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. No state reset, future score, interpolation or independent-window checkpoint. Availability includes source resampler buffering; score is the stored activity-projection proxy.",
|
|
"provenance": {
|
|
"source_export": "experiments/maai-ch-kyoto/results/user-only/score-timeline.json",
|
|
"source_export_sha256": "dd4959d14883746a13ab382e7a810d37e6d91014198ef53ff1e82e5ac6d61f5a",
|
|
"source_inference_provenance": {
|
|
"machine": "macOS-26.4.1-arm64-arm-64bit",
|
|
"hardware": "Apple M4 Mac mini / 16 GB",
|
|
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
|
|
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
|
|
"device": "cpu",
|
|
"threads": 4,
|
|
"dependencies": {
|
|
"maai": "0.2.22",
|
|
"torch": "2.14.1",
|
|
"transformers": "5.5.3",
|
|
"numpy": "2.4.6",
|
|
"soundfile": "0.14.0",
|
|
"soxr": "1.1.0"
|
|
},
|
|
"sample_rate": 16000,
|
|
"frame_ms": 100,
|
|
"context_ms": 20000,
|
|
"use_kv_cache": true,
|
|
"seed": 42,
|
|
"channel_mapping": {
|
|
"0": "original user",
|
|
"1": "all-zero robot input"
|
|
},
|
|
"weight_sha256": {
|
|
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
|
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
|
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
|
|
"script_sha256": "93c8a69850cd444c8daa9525eda267cda5f4ca34393df5f175286d875aa19195",
|
|
"model_factory_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe",
|
|
"loader_sha256": "4a7b37e6b67e5c55c030bcc399ab58e6575e49592dbda87dc311ec88da38e1d2",
|
|
"audio_sha256": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
|
}
|
|
},
|
|
"source_model_sha256": {},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"evaluation_script_sha256": "85d0b5f0f8dcf73cc6ee1b651e5a2cfc0b40e95024aacb0bd3ed975079ad9507"
|
|
},
|
|
"limitations": [
|
|
"Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.",
|
|
"Uses stored rounded continuous scores; no new inference or model-speed measurement.",
|
|
"No prior output or stale output is unavailable, not a negative/zero prediction.",
|
|
"Archived independent-window results are a separate evaluation protocol."
|
|
],
|
|
"accuracy_delta_pp_vs_cpc_stereo": -13.636363636363635,
|
|
"relative_accuracy_change_vs_cpc_stereo": -0.2
|
|
},
|
|
"maai_mimi_standard": {
|
|
"model": "maai_mimi_standard",
|
|
"name": "Mimi 标准双声道",
|
|
"evaluation_mode": "streaming",
|
|
"created_at": "2026-10-28T00:27:46.334666+00:00",
|
|
"threshold": 0.5,
|
|
"rule": "1 - p_now[user] > 0.5",
|
|
"main_metrics": {
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 17,
|
|
"fp": 5,
|
|
"fn": 8,
|
|
"tp": 14,
|
|
"correct": 31,
|
|
"accuracy": 0.7045454545454546,
|
|
"false_reply_rate": 0.22727272727272727,
|
|
"false_hold_rate": 0.36363636363636365,
|
|
"recall": 0.6363636363636364,
|
|
"precision": 0.7368421052631579,
|
|
"f1": 0.6829268292682927
|
|
},
|
|
"thresholds": [
|
|
{
|
|
"threshold": 0.11,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 8,
|
|
"fp": 14,
|
|
"fn": 1,
|
|
"tp": 21,
|
|
"correct": 29,
|
|
"accuracy": 0.6590909090909091,
|
|
"false_reply_rate": 0.6363636363636364,
|
|
"false_hold_rate": 0.045454545454545456,
|
|
"recall": 0.9545454545454546,
|
|
"precision": 0.6,
|
|
"f1": 0.7368421052631579
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 17,
|
|
"fp": 5,
|
|
"fn": 8,
|
|
"tp": 14,
|
|
"correct": 31,
|
|
"accuracy": 0.7045454545454546,
|
|
"false_reply_rate": 0.22727272727272727,
|
|
"false_hold_rate": 0.36363636363636365,
|
|
"recall": 0.6363636363636364,
|
|
"precision": 0.7368421052631579,
|
|
"f1": 0.6829268292682927
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 20,
|
|
"fp": 2,
|
|
"fn": 16,
|
|
"tp": 6,
|
|
"correct": 26,
|
|
"accuracy": 0.5909090909090909,
|
|
"false_reply_rate": 0.09090909090909091,
|
|
"false_hold_rate": 0.7272727272727273,
|
|
"recall": 0.2727272727272727,
|
|
"precision": 0.75,
|
|
"f1": 0.4
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 18,
|
|
"tp": 4,
|
|
"correct": 25,
|
|
"accuracy": 0.5681818181818182,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.8181818181818182,
|
|
"recall": 0.18181818181818182,
|
|
"precision": 0.8,
|
|
"f1": 0.2962962962962963
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 19,
|
|
"tp": 3,
|
|
"correct": 25,
|
|
"accuracy": 0.5681818181818182,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.8636363636363636,
|
|
"recall": 0.13636363636363635,
|
|
"precision": 1.0,
|
|
"f1": 0.24
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"correct": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.98,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"correct": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
}
|
|
],
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"n": 6,
|
|
"requested_labeled": 6,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 1,
|
|
"fn": 3,
|
|
"tp": 1,
|
|
"correct": 2,
|
|
"accuracy": 0.3333333333333333,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.75,
|
|
"recall": 0.25,
|
|
"precision": 0.5,
|
|
"f1": 0.3333333333333333
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"n": 8,
|
|
"requested_labeled": 8,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.75,
|
|
"f1": 0.75
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 2,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.6666666666666666,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.4,
|
|
"recall": 0.6,
|
|
"precision": 0.75,
|
|
"f1": 0.6666666666666666
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"n": 5,
|
|
"requested_labeled": 5,
|
|
"unavailable_labeled": 0,
|
|
"tn": 2,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 1,
|
|
"correct": 3,
|
|
"accuracy": 0.6,
|
|
"false_reply_rate": 0.3333333333333333,
|
|
"false_hold_rate": 0.5,
|
|
"recall": 0.5,
|
|
"precision": 0.5,
|
|
"f1": 0.5
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 5,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"correct": 8,
|
|
"accuracy": 0.8888888888888888,
|
|
"false_reply_rate": 0.16666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.75,
|
|
"f1": 0.8571428571428571
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"n": 7,
|
|
"requested_labeled": 7,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.8571428571428571,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 1.0,
|
|
"f1": 0.8571428571428571
|
|
}
|
|
},
|
|
"method": "Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. No state reset, future score, interpolation or independent-window checkpoint. Availability includes source resampler buffering; score is the stored activity-projection proxy.",
|
|
"provenance": {
|
|
"source_export": "experiments/maai-ch-kyoto/results/mimi-standard/score-timeline.json",
|
|
"source_export_sha256": "ffa50faef2d7809fc16a503793918221f580319c63dec9e13c7338d90fea6fd0",
|
|
"source_inference_provenance": {
|
|
"machine": "macOS-26.4.1-arm64-arm-64bit",
|
|
"hardware": "Apple M4 Mac mini / 16 GB",
|
|
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
|
|
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
|
|
"device": "cpu",
|
|
"torch_threads": 4,
|
|
"torch_interop_threads": 1,
|
|
"onnx_intra_threads": 4,
|
|
"onnx_inter_threads": 1,
|
|
"dependencies": {
|
|
"maai": "0.2.22",
|
|
"torch": "2.14.1",
|
|
"transformers": "5.5.3",
|
|
"onnxruntime": "1.31.0",
|
|
"numpy": "2.4.6",
|
|
"soundfile": "0.14.0",
|
|
"soxr": "1.1.0"
|
|
},
|
|
"model_type": "normal-ver2",
|
|
"variant": "vap",
|
|
"mc": false,
|
|
"checkpoint_filename": "vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"encoder": "mimi",
|
|
"encoder_precision": "fp32",
|
|
"channels": {
|
|
"0": "recorded user",
|
|
"1": "recorded robot"
|
|
},
|
|
"sample_rate": 16000,
|
|
"frame_rate": 12.5,
|
|
"frame_ms": 80,
|
|
"context_ms": 20000,
|
|
"seed": 42,
|
|
"use_kv_cache": true,
|
|
"weights": {
|
|
"files": [
|
|
{
|
|
"filename": "vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"repo": "maai-kyoto/vap_ch_kyoto",
|
|
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
|
|
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"sha256": "2820d2b172a6ddd4696d33edb43933f436a09a3e0d3eb41c7dc390c33b22d5ce",
|
|
"size": 15052341,
|
|
"upstream_blob_id": "cd3d8d053652e41ecfba747a193b98a230ffcbca"
|
|
},
|
|
{
|
|
"filename": "continuous_mimi_fp32.onnx",
|
|
"repo": "maai-kyoto/continuous-mimi-onnx",
|
|
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
|
|
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.onnx",
|
|
"sha256": "c86171a48e1bd46aa31a52e7fe00f74acc310e0d15d11881516cdd1df1639b54",
|
|
"size": 155877436,
|
|
"upstream_blob_id": "1ade42d5caa75f7567c2f904245cf893584e6ed9"
|
|
},
|
|
{
|
|
"filename": "continuous_mimi_fp32.json",
|
|
"repo": "maai-kyoto/continuous-mimi-onnx",
|
|
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
|
|
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.json",
|
|
"sha256": "221ab200381209d22ddf0291aff8fd0752633f0ddf7cbdd344cda6b039333c74",
|
|
"size": 7411,
|
|
"upstream_blob_id": "21be6fa62ec6c92cc9ba003c72ea09c218e9946e"
|
|
},
|
|
{
|
|
"filename": "vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"repo": "maai-kyoto/vap_ch_kyoto",
|
|
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
|
|
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"sha256": "61c9e186d765cb6c338cb7fdacb68baf4b2533e72cde7c803e52480a62919219",
|
|
"size": 15052584,
|
|
"upstream_blob_id": "cd6e6701d6c5fc7a659a82952e92ce2569b7b111"
|
|
}
|
|
]
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
|
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
|
|
"script_sha256": "36b950aa396e6008bbd6e32410902dd2859f68b17b9f8fb5691a97d415e80993",
|
|
"maai_encoder_source_sha256": "80f5d928ec48c7839c2e935043df0e1c82b523096834f6cbc968ccb45ddc6fbd",
|
|
"maai_model_source_sha256": "824730f564f0da18d0d4c19df1a376b692546732faf8eb7aaf871d74480ff504",
|
|
"audio_sha256": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
|
},
|
|
"cpc_reference_predictions_sha256": "266f43c91d232e5fa7e8e9fc10c294c8f7e7df64fac2332a6a7441faf15d30a8"
|
|
},
|
|
"source_model_sha256": {},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"evaluation_script_sha256": "85d0b5f0f8dcf73cc6ee1b651e5a2cfc0b40e95024aacb0bd3ed975079ad9507"
|
|
},
|
|
"limitations": [
|
|
"Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.",
|
|
"Uses stored rounded continuous scores; no new inference or model-speed measurement.",
|
|
"No prior output or stale output is unavailable, not a negative/zero prediction.",
|
|
"Archived independent-window results are a separate evaluation protocol."
|
|
],
|
|
"accuracy_delta_pp_vs_cpc_stereo": 2.272727272727282,
|
|
"relative_accuracy_change_vs_cpc_stereo": 0.03333333333333347
|
|
},
|
|
"maai_mimi_mc": {
|
|
"model": "maai_mimi_mc",
|
|
"name": "Mimi 降噪双声道",
|
|
"evaluation_mode": "streaming",
|
|
"created_at": "2026-10-28T00:27:46.346979+00:00",
|
|
"threshold": 0.5,
|
|
"rule": "1 - p_now[user] > 0.5",
|
|
"main_metrics": {
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 17,
|
|
"fp": 5,
|
|
"fn": 5,
|
|
"tp": 17,
|
|
"correct": 34,
|
|
"accuracy": 0.7727272727272727,
|
|
"false_reply_rate": 0.22727272727272727,
|
|
"false_hold_rate": 0.22727272727272727,
|
|
"recall": 0.7727272727272727,
|
|
"precision": 0.7727272727272727,
|
|
"f1": 0.7727272727272727
|
|
},
|
|
"thresholds": [
|
|
{
|
|
"threshold": 0.11,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 8,
|
|
"fp": 14,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"correct": 30,
|
|
"accuracy": 0.6818181818181818,
|
|
"false_reply_rate": 0.6363636363636364,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.6111111111111112,
|
|
"f1": 0.7586206896551724
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 17,
|
|
"fp": 5,
|
|
"fn": 5,
|
|
"tp": 17,
|
|
"correct": 34,
|
|
"accuracy": 0.7727272727272727,
|
|
"false_reply_rate": 0.22727272727272727,
|
|
"false_hold_rate": 0.22727272727272727,
|
|
"recall": 0.7727272727272727,
|
|
"precision": 0.7727272727272727,
|
|
"f1": 0.7727272727272727
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 19,
|
|
"fp": 3,
|
|
"fn": 13,
|
|
"tp": 9,
|
|
"correct": 28,
|
|
"accuracy": 0.6363636363636364,
|
|
"false_reply_rate": 0.13636363636363635,
|
|
"false_hold_rate": 0.5909090909090909,
|
|
"recall": 0.4090909090909091,
|
|
"precision": 0.75,
|
|
"f1": 0.5294117647058824
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 15,
|
|
"tp": 7,
|
|
"correct": 28,
|
|
"accuracy": 0.6363636363636364,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.6818181818181818,
|
|
"recall": 0.3181818181818182,
|
|
"precision": 0.875,
|
|
"f1": 0.4666666666666667
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 18,
|
|
"tp": 4,
|
|
"correct": 25,
|
|
"accuracy": 0.5681818181818182,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.8181818181818182,
|
|
"recall": 0.18181818181818182,
|
|
"precision": 0.8,
|
|
"f1": 0.2962962962962963
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 20,
|
|
"tp": 2,
|
|
"correct": 24,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9090909090909091,
|
|
"recall": 0.09090909090909091,
|
|
"precision": 1.0,
|
|
"f1": 0.16666666666666666
|
|
},
|
|
{
|
|
"threshold": 0.98,
|
|
"n": 44,
|
|
"requested_labeled": 44,
|
|
"unavailable_labeled": 0,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 22,
|
|
"tp": 0,
|
|
"correct": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 1.0,
|
|
"recall": 0.0,
|
|
"precision": null,
|
|
"f1": 0.0
|
|
}
|
|
],
|
|
"per_call": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
|
"n": 6,
|
|
"requested_labeled": 6,
|
|
"unavailable_labeled": 0,
|
|
"tn": 1,
|
|
"fp": 1,
|
|
"fn": 3,
|
|
"tp": 1,
|
|
"correct": 2,
|
|
"accuracy": 0.3333333333333333,
|
|
"false_reply_rate": 0.5,
|
|
"false_hold_rate": 0.75,
|
|
"recall": 0.25,
|
|
"precision": 0.5,
|
|
"f1": 0.3333333333333333
|
|
},
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
|
"n": 8,
|
|
"requested_labeled": 8,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 0.75,
|
|
"f1": 0.75
|
|
},
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 5,
|
|
"correct": 8,
|
|
"accuracy": 0.8888888888888888,
|
|
"false_reply_rate": 0.25,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.8333333333333334,
|
|
"f1": 0.9090909090909091
|
|
},
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
|
"n": 5,
|
|
"requested_labeled": 5,
|
|
"unavailable_labeled": 0,
|
|
"tn": 2,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 2,
|
|
"correct": 4,
|
|
"accuracy": 0.8,
|
|
"false_reply_rate": 0.3333333333333333,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.6666666666666666,
|
|
"f1": 0.8
|
|
},
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
|
"n": 9,
|
|
"requested_labeled": 9,
|
|
"unavailable_labeled": 0,
|
|
"tn": 5,
|
|
"fp": 1,
|
|
"fn": 0,
|
|
"tp": 3,
|
|
"correct": 8,
|
|
"accuracy": 0.8888888888888888,
|
|
"false_reply_rate": 0.16666666666666666,
|
|
"false_hold_rate": 0.0,
|
|
"recall": 1.0,
|
|
"precision": 0.75,
|
|
"f1": 0.8571428571428571
|
|
},
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
|
"n": 7,
|
|
"requested_labeled": 7,
|
|
"unavailable_labeled": 0,
|
|
"tn": 3,
|
|
"fp": 0,
|
|
"fn": 1,
|
|
"tp": 3,
|
|
"correct": 6,
|
|
"accuracy": 0.8571428571428571,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.25,
|
|
"recall": 0.75,
|
|
"precision": 1.0,
|
|
"f1": 0.8571428571428571
|
|
}
|
|
},
|
|
"method": "Continuous full-call VAP outputs only. At each exact decision_ms, use the last score with availability timestamp <= decision_ms and age <= 2 * sample_interval_ms. No state reset, future score, interpolation or independent-window checkpoint. Availability includes source resampler buffering; score is the stored activity-projection proxy.",
|
|
"provenance": {
|
|
"source_export": "experiments/maai-ch-kyoto/results/mimi-mc/score-timeline.json",
|
|
"source_export_sha256": "e080ecf2df7c65b7d4a43c3bdabd9b2a0026f3c9ffcfd086da124edba051d355",
|
|
"source_inference_provenance": {
|
|
"machine": "macOS-26.4.1-arm64-arm-64bit",
|
|
"hardware": "Apple M4 Mac mini / 16 GB",
|
|
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
|
|
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
|
|
"device": "cpu",
|
|
"torch_threads": 4,
|
|
"torch_interop_threads": 1,
|
|
"onnx_intra_threads": 4,
|
|
"onnx_inter_threads": 1,
|
|
"dependencies": {
|
|
"maai": "0.2.22",
|
|
"torch": "2.14.1",
|
|
"transformers": "5.5.3",
|
|
"onnxruntime": "1.31.0",
|
|
"numpy": "2.4.6",
|
|
"soundfile": "0.14.0",
|
|
"soxr": "1.1.0"
|
|
},
|
|
"model_type": "normal-ver2",
|
|
"variant": "vap_mc",
|
|
"mc": true,
|
|
"checkpoint_filename": "vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"encoder": "mimi",
|
|
"encoder_precision": "fp32",
|
|
"channels": {
|
|
"0": "recorded user",
|
|
"1": "recorded robot"
|
|
},
|
|
"sample_rate": 16000,
|
|
"frame_rate": 12.5,
|
|
"frame_ms": 80,
|
|
"context_ms": 20000,
|
|
"seed": 42,
|
|
"use_kv_cache": true,
|
|
"weights": {
|
|
"files": [
|
|
{
|
|
"filename": "vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"repo": "maai-kyoto/vap_ch_kyoto",
|
|
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
|
|
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"sha256": "2820d2b172a6ddd4696d33edb43933f436a09a3e0d3eb41c7dc390c33b22d5ce",
|
|
"size": 15052341,
|
|
"upstream_blob_id": "cd3d8d053652e41ecfba747a193b98a230ffcbca"
|
|
},
|
|
{
|
|
"filename": "continuous_mimi_fp32.onnx",
|
|
"repo": "maai-kyoto/continuous-mimi-onnx",
|
|
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
|
|
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.onnx",
|
|
"sha256": "c86171a48e1bd46aa31a52e7fe00f74acc310e0d15d11881516cdd1df1639b54",
|
|
"size": 155877436,
|
|
"upstream_blob_id": "1ade42d5caa75f7567c2f904245cf893584e6ed9"
|
|
},
|
|
{
|
|
"filename": "continuous_mimi_fp32.json",
|
|
"repo": "maai-kyoto/continuous-mimi-onnx",
|
|
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
|
|
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.json",
|
|
"sha256": "221ab200381209d22ddf0291aff8fd0752633f0ddf7cbdd344cda6b039333c74",
|
|
"size": 7411,
|
|
"upstream_blob_id": "21be6fa62ec6c92cc9ba003c72ea09c218e9946e"
|
|
},
|
|
{
|
|
"filename": "vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"repo": "maai-kyoto/vap_ch_kyoto",
|
|
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
|
|
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
|
|
"sha256": "61c9e186d765cb6c338cb7fdacb68baf4b2533e72cde7c803e52480a62919219",
|
|
"size": 15052584,
|
|
"upstream_blob_id": "cd6e6701d6c5fc7a659a82952e92ce2569b7b111"
|
|
}
|
|
]
|
|
},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
|
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
|
|
"script_sha256": "36b950aa396e6008bbd6e32410902dd2859f68b17b9f8fb5691a97d415e80993",
|
|
"maai_encoder_source_sha256": "80f5d928ec48c7839c2e935043df0e1c82b523096834f6cbc968ccb45ddc6fbd",
|
|
"maai_model_source_sha256": "824730f564f0da18d0d4c19df1a376b692546732faf8eb7aaf871d74480ff504",
|
|
"audio_sha256": {
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
|
},
|
|
"cpc_reference_predictions_sha256": "266f43c91d232e5fa7e8e9fc10c294c8f7e7df64fac2332a6a7441faf15d30a8"
|
|
},
|
|
"source_model_sha256": {},
|
|
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
|
|
"evaluation_script_sha256": "85d0b5f0f8dcf73cc6ee1b651e5a2cfc0b40e95024aacb0bd3ed975079ad9507"
|
|
},
|
|
"limitations": [
|
|
"Selected-point classification on six recorded calls, not endpointing-policy cutoff/latency.",
|
|
"Uses stored rounded continuous scores; no new inference or model-speed measurement.",
|
|
"No prior output or stale output is unavailable, not a negative/zero prediction.",
|
|
"Archived independent-window results are a separate evaluation protocol."
|
|
],
|
|
"accuracy_delta_pp_vs_cpc_stereo": 9.090909090909093,
|
|
"relative_accuracy_change_vs_cpc_stereo": 0.1333333333333334
|
|
}
|
|
}
|
|
}
|