wx44wx/xiaobailing-turn-detection
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
6785e3a07b2def71d11f8f63ee43a0b1ed2c34c9
xiaobailing-turn-detection/experiments/maai-ch-kyoto/results/mimi-mc/summary.json
T

367 lines
13 KiB
JSON
Raw Blame History

{
"model": "VAP / MaAI Mimi mc (both speakers)",
"created_at": "2026-10-27T07:12:05.611060+00:00",
"total_points": 44,
"main_rule": "1 - p_now[user] > 0.5",
"main_metrics": {
"n": 44,
"tn": 15,
"fp": 7,
"fn": 3,
"tp": 19,
"accuracy": 0.7727272727272727,
"false_reply_rate": 0.3181818181818182,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.7307692307692307,
"f1": 0.7916666666666666
},
"cpc_mc_stereo_reference": {
"n": 44,
"tn": 14,
"fp": 8,
"fn": 4,
"tp": 18,
"accuracy": 0.7272727272727273,
"false_reply_rate": 0.36363636363636365,
"false_hold_rate": 0.18181818181818182,
"recall": 0.8181818181818182,
"precision": 0.6923076923076923,
"f1": 0.75
},
"cpc_mc_user_only_reference": {
"n": 44,
"tn": 4,
"fp": 18,
"fn": 3,
"tp": 19,
"accuracy": 0.5227272727272727,
"false_reply_rate": 0.8181818181818182,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.5135135135135135,
"f1": 0.6440677966101694
},
"thresholds": [
{
"threshold": 0.11,
"n": 44,
"tn": 8,
"fp": 14,
"fn": 0,
"tp": 22,
"accuracy": 0.6818181818181818,
"false_reply_rate": 0.6363636363636364,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.6111111111111112,
"f1": 0.7586206896551724
},
{
"threshold": 0.5,
"n": 44,
"tn": 15,
"fp": 7,
"fn": 3,
"tp": 19,
"accuracy": 0.7727272727272727,
"false_reply_rate": 0.3181818181818182,
"false_hold_rate": 0.13636363636363635,
"recall": 0.8636363636363636,
"precision": 0.7307692307692307,
"f1": 0.7916666666666666
},
{
"threshold": 0.7,
"n": 44,
"tn": 20,
"fp": 2,
"fn": 7,
"tp": 15,
"accuracy": 0.7954545454545454,
"false_reply_rate": 0.09090909090909091,
"false_hold_rate": 0.3181818181818182,
"recall": 0.6818181818181818,
"precision": 0.8823529411764706,
"f1": 0.7692307692307693
},
{
"threshold": 0.8,
"n": 44,
"tn": 20,
"fp": 2,
"fn": 11,
"tp": 11,
"accuracy": 0.7045454545454546,
"false_reply_rate": 0.09090909090909091,
"false_hold_rate": 0.5,
"recall": 0.5,
"precision": 0.8461538461538461,
"f1": 0.6285714285714286
},
{
"threshold": 0.9,
"n": 44,
"tn": 21,
"fp": 1,
"fn": 15,
"tp": 7,
"accuracy": 0.6363636363636364,
"false_reply_rate": 0.045454545454545456,
"false_hold_rate": 0.6818181818181818,
"recall": 0.3181818181818182,
"precision": 0.875,
"f1": 0.4666666666666667
},
{
"threshold": 0.95,
"n": 44,
"tn": 22,
"fp": 0,
"fn": 17,
"tp": 5,
"accuracy": 0.6136363636363636,
"false_reply_rate": 0.0,
"false_hold_rate": 0.7727272727272727,
"recall": 0.22727272727272727,
"precision": 1.0,
"f1": 0.37037037037037035
},
{
"threshold": 0.98,
"n": 44,
"tn": 22,
"fp": 0,
"fn": 22,
"tp": 0,
"accuracy": 0.5,
"false_reply_rate": 0.0,
"false_hold_rate": 1.0,
"recall": 0.0,
"precision": null,
"f1": 0.0
}
],
"per_call": {
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
"n": 6,
"tn": 1,
"fp": 1,
"fn": 1,
"tp": 3,
"accuracy": 0.6666666666666666,
"false_reply_rate": 0.5,
"false_hold_rate": 0.25,
"recall": 0.75,
"precision": 0.75,
"f1": 0.75
},
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
"n": 8,
"tn": 3,
"fp": 1,
"fn": 1,
"tp": 3,
"accuracy": 0.75,
"false_reply_rate": 0.25,
"false_hold_rate": 0.25,
"recall": 0.75,
"precision": 0.75,
"f1": 0.75
},
"872a1e67-873a-4c49-a598-b38a670afcad": {
"n": 9,
"tn": 3,
"fp": 1,
"fn": 1,
"tp": 4,
"accuracy": 0.7777777777777778,
"false_reply_rate": 0.25,
"false_hold_rate": 0.2,
"recall": 0.8,
"precision": 0.8,
"f1": 0.8
},
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
"n": 5,
"tn": 1,
"fp": 2,
"fn": 0,
"tp": 2,
"accuracy": 0.6,
"false_reply_rate": 0.6666666666666666,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.5,
"f1": 0.6666666666666666
},
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
"n": 9,
"tn": 4,
"fp": 2,
"fn": 0,
"tp": 3,
"accuracy": 0.7777777777777778,
"false_reply_rate": 0.3333333333333333,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 0.6,
"f1": 0.75
},
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
"n": 7,
"tn": 3,
"fp": 0,
"fn": 0,
"tp": 4,
"accuracy": 1.0,
"false_reply_rate": 0.0,
"false_hold_rate": 0.0,
"recall": 1.0,
"precision": 1.0,
"f1": 1.0
}
},
"changed_vs_cpc_mc_stereo": {
"fixed": 5,
"regressed": 3
},
"annotation_sources": {
"xlsx_draft": 29,
"unspecified": 15
},
"provenance": {
"machine": "macOS-26.4.1-arm64-arm-64bit",
"hardware": "Apple M4 Mac mini / 16 GB",
"python": "3.11.15 (main, Mar 25 2026, 02:58:47) [Clang 22.1.1 ]",
"environment": "experiments/maai-ch-kyoto/.venv (uv.lock)",
"device": "cpu",
"torch_threads": 4,
"torch_interop_threads": 1,
"onnx_intra_threads": 4,
"onnx_inter_threads": 1,
"dependencies": {
"maai": "0.2.22",
"torch": "2.14.1",
"transformers": "5.5.3",
"onnxruntime": "1.31.0",
"numpy": "2.4.6",
"soundfile": "0.14.0",
"soxr": "1.1.0"
},
"model_type": "normal-ver2",
"variant": "vap_mc",
"mc": true,
"checkpoint_filename": "vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
"encoder": "mimi",
"encoder_precision": "fp32",
"channels": {
"0": "recorded user",
"1": "recorded robot"
},
"sample_rate": 16000,
"frame_rate": 12.5,
"frame_ms": 80,
"context_ms": 20000,
"seed": 42,
"use_kv_cache": true,
"weights": {
"files": [
{
"filename": "vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
"repo": "maai-kyoto/vap_ch_kyoto",
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
"sha256": "2820d2b172a6ddd4696d33edb43933f436a09a3e0d3eb41c7dc390c33b22d5ce",
"size": 15052341,
"upstream_blob_id": "cd3d8d053652e41ecfba747a193b98a230ffcbca"
},
{
"filename": "continuous_mimi_fp32.onnx",
"repo": "maai-kyoto/continuous-mimi-onnx",
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.onnx",
"sha256": "c86171a48e1bd46aa31a52e7fe00f74acc310e0d15d11881516cdd1df1639b54",
"size": 155877436,
"upstream_blob_id": "1ade42d5caa75f7567c2f904245cf893584e6ed9"
},
{
"filename": "continuous_mimi_fp32.json",
"repo": "maai-kyoto/continuous-mimi-onnx",
"revision": "303ef4ec0c70e4c71b67ea166acd26dd0d96655c",
"url": "https://huggingface.co/maai-kyoto/continuous-mimi-onnx/resolve/303ef4ec0c70e4c71b67ea166acd26dd0d96655c/continuous_mimi_fp32.json",
"sha256": "221ab200381209d22ddf0291aff8fd0752633f0ddf7cbdd344cda6b039333c74",
"size": 7411,
"upstream_blob_id": "21be6fa62ec6c92cc9ba003c72ea09c218e9946e"
},
{
"filename": "vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
"repo": "maai-kyoto/vap_ch_kyoto",
"revision": "c135cdcfa12ff5aa1cce2fdd23df110941a78f38",
"url": "https://huggingface.co/maai-kyoto/vap_ch_kyoto/resolve/c135cdcfa12ff5aa1cce2fdd23df110941a78f38/vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt",
"sha256": "61c9e186d765cb6c338cb7fdacb68baf4b2533e72cde7c803e52480a62919219",
"size": 15052584,
"upstream_blob_id": "cd6e6701d6c5fc7a659a82952e92ce2569b7b111"
}
]
},
"label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
"uv_lock_sha256": "6863631a5969e0f077d8aefc4335532952fe95486598b609e832e36d5ce2777f",
"script_sha256": "36b950aa396e6008bbd6e32410902dd2859f68b17b9f8fb5691a97d415e80993",
"maai_encoder_source_sha256": "80f5d928ec48c7839c2e935043df0e1c82b523096834f6cbc968ccb45ddc6fbd",
"maai_model_source_sha256": "824730f564f0da18d0d4c19df1a376b692546732faf8eb7aaf871d74480ff504",
"audio_sha256": {
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
},
"cpc_reference_predictions_sha256": "266f43c91d232e5fa7e8e9fc10c294c8f7e7df64fac2332a6a7441faf15d30a8"
},
"witness": {
"repeat_identical": true,
"future_invariance": true,
"both_channels_nonzero": true,
"score": 0.965790644288063,
"timing": {
"model_load_ms": 43.37858222424984,
"replay_inference_ms": 3893.7117541208863,
"mean_update_ms": 15.574847016483545,
"p95_update_ms": 18.14010818488896,
"first_output_ms": 160.0,
"frame_count": 250
},
"loading_audit": {
"variant": "vap_mc",
"loader_architecture_marker": "vap",
"encoder": "mimi",
"checkpoint_mc": true,
"single_tower": false,
"missing_trainable_parameters": [],
"mismatched_parameters": [],
"unused_frozen_frame_rate_parameters": [
"encoder1.frame_rate_conv.weight",
"encoder1.frame_rate_conv.bias",
"encoder2.frame_rate_conv.weight",
"encoder2.frame_rate_conv.bias"
],
"loader_messages": "Loading model from local file: /Users/wangx/Code/晓白灵智能打断/experiments/maai-ch-kyoto/models/mimi/vap_mc_mimi_state_dict_ch_kyoto_12.5hz_20000msec.pt\nUsing ONNX Mimi backend (fp32, local): /Users/wangx/Code/晓白灵智能打断/experiments/maai-ch-kyoto/models/mimi/continuous_mimi_fp32.onnx\nFroze EncoderMimiOnnx!\nUsing ONNX Mimi backend (fp32, local): /Users/wangx/Code/晓白灵智能打断/experiments/maai-ch-kyoto/models/mimi/continuous_mimi_fp32.onnx\nFroze EncoderMimiOnnx!\nfreeze encoder\nFroze EncoderMimiOnnx!\nFroze EncoderMimiOnnx!\nWarning: Parameter 'ar_channel.layers.0.mha.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.0.mha.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.0.mha_cross.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.1.mha.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.1.mha_cross.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.2.mha.m' was not updated from its initial value.\nWarning: Parameter 'ar.layers.2.mha_cross.m' was not updated from its initial value.\nWarning: Parameter 'objective.codebook.emb.weight' was not updated from its initial value.\n",
"encoder_provider": [
"CPUExecutionProvider"
]
}
},
"method": "mc (mc=True) ch_kyoto VAP with official FP32 ONNX Mimi encoder. Recorded user+robot stereo; crop preceding <=20s before 16kHz resampling; 80ms native frames, final frame ends at exact decision. Reset Mimi and VAP state per point, retain official skipped startup encoder output. No future input, annotation text, external end signal, silence gate or forced timeout. Score=1-p_now[user], not calibrated EOT probability.",
"limitations": [
"Comparison changes encoder/checkpoints and native framing; no held-out benchmark or claim that differences are caused by the encoder alone.",
"Both models use <=20s histories but native 80ms/100ms framing may differ at recording start by less than one frame.",
"Continuous full-call states differ from independent point replay; checkpoint scores need not lie on the continuous curve.",
"Replay computation includes the entire history; it is not one-step online update latency."
],
"timeline_points": 4867,
"elapsed_seconds": 278.2951404163614
}
Reference in New Issue View Git Blame Copy Permalink