新增 Mimi 与仅用户 VAP 对比并统一流式评测

2026-10-28 22:44:36 +08:00
parent 19d4e3c767
commit 6785e3a07b
55 changed files with 6160 additions and 31 deletions
@@ -0,0 +1,45 @@
+call_id,decision_ms,can_reply_now,mimi_standard,mimi_mc,cpc_mc_stereo,cpc_mc_user_only
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,21780,1,0.9191182181239128,0.965790644288063,0.959835909307003,0.9077615067362785
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,29860,1,0.9219910651445389,0.9577881693840027,0.9285890087485313,0.8796060606837273
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,48125,0,0.46547502279281616,0.45545458793640137,0.5122989416122437,0.712736040353775
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,64000,1,0.7919017523527145,0.8706666529178619,0.9369851648807526,0.9320666417479515
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,76672,0,0.5827796757221222,0.6864748597145081,0.7066852748394012,0.8640106171369553
+2c83253a-7977-4e3c-a387-0aa1ea06fbe1,88482,1,0.2125949263572693,0.2585999369621277,0.11465108394622803,0.16062873601913452
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,22007,1,0.7384836673736572,0.764378547668457,0.69324991106987,0.9323981329798698
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,31842,1,0.48968255519866943,0.6238378286361694,0.7899426519870758,0.9342073872685432
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,48536,0,0.27077269554138184,0.43519163131713867,0.5186727344989777,0.8199913799762726
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,52348,1,0.8557324260473251,0.8981894701719284,0.959786981344223,0.9645285122096539
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,65298,0,0.021075308322906494,0.06646430492401123,0.16463828086853027,0.8862891793251038
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,67943,0,0.08532929420471191,0.05880063772201538,0.1254715919494629,0.6981820166110992
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,70430,0,0.9156609252095222,0.9186511859297752,0.9451650716364384,0.03903418779373169
+40ae686c-8196-4d7b-b7ad-3e12a427ebb6,82941,1,0.13118845224380493,0.295051634311676,0.27139031887054443,0.9212324172258377
+872a1e67-873a-4c49-a598-b38a670afcad,17153,1,0.3544103503227234,0.5796335935592651,0.4008920192718506,0.8734219521284103
+872a1e67-873a-4c49-a598-b38a670afcad,24846,1,0.43358469009399414,0.4162102937698364,0.6936855018138885,0.9015571922063828
+872a1e67-873a-4c49-a598-b38a670afcad,33471,1,0.4404107332229614,0.7417943477630615,0.802086129784584,0.929821141064167
+872a1e67-873a-4c49-a598-b38a670afcad,44810,0,0.385736346244812,0.3360837697982788,0.3700411319732666,0.8325859755277634
+872a1e67-873a-4c49-a598-b38a670afcad,45581,0,0.014162182807922363,0.015885531902313232,0.023214280605316162,0.2830030918121338
+872a1e67-873a-4c49-a598-b38a670afcad,47187,0,0.23951458930969238,0.14240401983261108,0.7005376219749451,0.8951215818524361
+872a1e67-873a-4c49-a598-b38a670afcad,49500,1,0.9564755298197269,0.9701901488006115,0.9640973135828972,0.4527696967124939
+872a1e67-873a-4c49-a598-b38a670afcad,52326,0,0.5608597099781036,0.5535028576850891,0.4240996241569519,0.6631686389446259
+872a1e67-873a-4c49-a598-b38a670afcad,54318,1,0.923841305077076,0.9644164405763149,0.9336890131235123,0.48515671491622925
+a266b9ef-5e47-4434-a04c-ce4eacd52c19,18027,1,0.8423347920179367,0.8495909720659256,0.5773889422416687,0.9133674055337906
+a266b9ef-5e47-4434-a04c-ce4eacd52c19,24768,1,0.429895281791687,0.8364158272743225,0.5596272945404053,0.8936084657907486
+a266b9ef-5e47-4434-a04c-ce4eacd52c19,41219,0,0.5353022813796997,0.5108224153518677,0.9102829173207283,0.9391313791275024
+a266b9ef-5e47-4434-a04c-ce4eacd52c19,51080,0,0.006074726581573486,0.007267594337463379,0.010066509246826172,0.0655754804611206
+a266b9ef-5e47-4434-a04c-ce4eacd52c19,68313,0,0.7153808772563934,0.8308001905679703,0.8074509650468826,0.8763869255781174
+cb3238dc-3946-42a5-aad8-9b9a54226363,13540,1,0.834052786231041,0.9350477233529091,0.9443961046636105,0.920970730483532
+cb3238dc-3946-42a5-aad8-9b9a54226363,19620,1,0.8876387625932693,0.953842855989933,0.9730777312070131,0.9391433596611023
+cb3238dc-3946-42a5-aad8-9b9a54226363,27520,1,0.9249144271016121,0.932467520236969,0.9681989774107933,0.9300618022680283
+cb3238dc-3946-42a5-aad8-9b9a54226363,41207,0,0.012665152549743652,0.014370322227478027,0.12327557802200317,0.7711529731750488
+cb3238dc-3946-42a5-aad8-9b9a54226363,49204,0,0.7771151214838028,0.632370263338089,0.8600987195968628,0.9226682409644127
+cb3238dc-3946-42a5-aad8-9b9a54226363,58424,0,0.14501017332077026,0.11090946197509766,0.19378310441970825,0.7652575969696045
+cb3238dc-3946-42a5-aad8-9b9a54226363,60790,0,0.006374657154083252,0.012694478034973145,0.05659288167953491,0.4327901005744934
+cb3238dc-3946-42a5-aad8-9b9a54226363,70745,0,0.14832377433776855,0.1670762300491333,0.28872406482696533,0.6845456063747406
+cb3238dc-3946-42a5-aad8-9b9a54226363,73640,0,0.5420903265476227,0.6731171607971191,0.45827627182006836,0.6812975704669952
+e304f740-85d4-427e-9fd6-c5474e9cd737,11531,1,0.6417741775512695,0.7890582382678986,0.5200230777263641,0.936267301440239
+e304f740-85d4-427e-9fd6-c5474e9cd737,18842,1,0.7854653149843216,0.7565313577651978,0.6192943453788757,0.877316839993
+e304f740-85d4-427e-9fd6-c5474e9cd737,29619,1,0.6686168611049652,0.6641051769256592,0.44960129261016846,0.8809438720345497
+e304f740-85d4-427e-9fd6-c5474e9cd737,40701,0,0.24793684482574463,0.43508362770080566,0.4185728430747986,0.8340967893600464
+e304f740-85d4-427e-9fd6-c5474e9cd737,54600,1,0.6685410737991333,0.6717596650123596,0.7056282758712769,0.8766583725810051
+e304f740-85d4-427e-9fd6-c5474e9cd737,62828,0,0.051033079624176025,0.06967586278915405,0.02205902338027954,0.5078669488430023
+e304f740-85d4-427e-9fd6-c5474e9cd737,65819,0,0.029323875904083252,0.029637634754180908,0.05253005027770996,0.6574011743068695
@@ -0,0 +1,377 @@
+{
+  "threshold": 0.5,
+  "rule": "1 - p_now[user] > 0.5",
+  "n": 44,
+  "positive": 22,
+  "models": {
+    "cpc_mc_stereo": {
+      "n": 44,
+      "correct": 32,
+      "tn": 14,
+      "fp": 8,
+      "fn": 4,
+      "tp": 18,
+      "accuracy": 0.7272727272727273,
+      "false_reply_rate": 0.36363636363636365,
+      "false_hold_rate": 0.18181818181818182,
+      "accuracy_delta_pp_vs_cpc_mc_stereo": 0.0,
+      "relative_accuracy_change_vs_cpc_mc_stereo": 0.0
+    },
+    "cpc_mc_user_only": {
+      "n": 44,
+      "correct": 23,
+      "tn": 4,
+      "fp": 18,
+      "fn": 3,
+      "tp": 19,
+      "accuracy": 0.5227272727272727,
+      "false_reply_rate": 0.8181818181818182,
+      "false_hold_rate": 0.13636363636363635,
+      "accuracy_delta_pp_vs_cpc_mc_stereo": -20.45454545454546,
+      "relative_accuracy_change_vs_cpc_mc_stereo": -0.28125000000000006
+    },
+    "mimi_standard": {
+      "n": 44,
+      "correct": 30,
+      "tn": 15,
+      "fp": 7,
+      "fn": 7,
+      "tp": 15,
+      "accuracy": 0.6818181818181818,
+      "false_reply_rate": 0.3181818181818182,
+      "false_hold_rate": 0.3181818181818182,
+      "accuracy_delta_pp_vs_cpc_mc_stereo": -4.545454545454552,
+      "relative_accuracy_change_vs_cpc_mc_stereo": -0.0625000000000001
+    },
+    "mimi_mc": {
+      "n": 44,
+      "correct": 34,
+      "tn": 15,
+      "fp": 7,
+      "fn": 3,
+      "tp": 19,
+      "accuracy": 0.7727272727272727,
+      "false_reply_rate": 0.3181818181818182,
+      "false_hold_rate": 0.13636363636363635,
+      "accuracy_delta_pp_vs_cpc_mc_stereo": 4.545454545454541,
+      "relative_accuracy_change_vs_cpc_mc_stereo": 0.062499999999999944
+    }
+  },
+  "paired_vs_cpc_mc_stereo": {
+    "mimi_standard": {
+      "fixed": 4,
+      "regressed": 6
+    },
+    "mimi_mc": {
+      "fixed": 5,
+      "regressed": 3
+    }
+  },
+  "per_call": {
+    "2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
+      "cpc_mc_stereo": {
+        "n": 6,
+        "correct": 3,
+        "tn": 0,
+        "fp": 2,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.5,
+        "false_reply_rate": 1.0,
+        "false_hold_rate": 0.25
+      },
+      "cpc_mc_user_only": {
+        "n": 6,
+        "correct": 3,
+        "tn": 0,
+        "fp": 2,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.5,
+        "false_reply_rate": 1.0,
+        "false_hold_rate": 0.25
+      },
+      "mimi_standard": {
+        "n": 6,
+        "correct": 4,
+        "tn": 1,
+        "fp": 1,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.6666666666666666,
+        "false_reply_rate": 0.5,
+        "false_hold_rate": 0.25
+      },
+      "mimi_mc": {
+        "n": 6,
+        "correct": 4,
+        "tn": 1,
+        "fp": 1,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.6666666666666666,
+        "false_reply_rate": 0.5,
+        "false_hold_rate": 0.25
+      }
+    },
+    "40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
+      "cpc_mc_stereo": {
+        "n": 8,
+        "correct": 5,
+        "tn": 2,
+        "fp": 2,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.625,
+        "false_reply_rate": 0.5,
+        "false_hold_rate": 0.25
+      },
+      "cpc_mc_user_only": {
+        "n": 8,
+        "correct": 5,
+        "tn": 1,
+        "fp": 3,
+        "fn": 0,
+        "tp": 4,
+        "accuracy": 0.625,
+        "false_reply_rate": 0.75,
+        "false_hold_rate": 0.0
+      },
+      "mimi_standard": {
+        "n": 8,
+        "correct": 5,
+        "tn": 3,
+        "fp": 1,
+        "fn": 2,
+        "tp": 2,
+        "accuracy": 0.625,
+        "false_reply_rate": 0.25,
+        "false_hold_rate": 0.5
+      },
+      "mimi_mc": {
+        "n": 8,
+        "correct": 6,
+        "tn": 3,
+        "fp": 1,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.75,
+        "false_reply_rate": 0.25,
+        "false_hold_rate": 0.25
+      }
+    },
+    "872a1e67-873a-4c49-a598-b38a670afcad": {
+      "cpc_mc_stereo": {
+        "n": 9,
+        "correct": 7,
+        "tn": 3,
+        "fp": 1,
+        "fn": 1,
+        "tp": 4,
+        "accuracy": 0.7777777777777778,
+        "false_reply_rate": 0.25,
+        "false_hold_rate": 0.2
+      },
+      "cpc_mc_user_only": {
+        "n": 9,
+        "correct": 4,
+        "tn": 1,
+        "fp": 3,
+        "fn": 2,
+        "tp": 3,
+        "accuracy": 0.4444444444444444,
+        "false_reply_rate": 0.75,
+        "false_hold_rate": 0.4
+      },
+      "mimi_standard": {
+        "n": 9,
+        "correct": 5,
+        "tn": 3,
+        "fp": 1,
+        "fn": 3,
+        "tp": 2,
+        "accuracy": 0.5555555555555556,
+        "false_reply_rate": 0.25,
+        "false_hold_rate": 0.6
+      },
+      "mimi_mc": {
+        "n": 9,
+        "correct": 7,
+        "tn": 3,
+        "fp": 1,
+        "fn": 1,
+        "tp": 4,
+        "accuracy": 0.7777777777777778,
+        "false_reply_rate": 0.25,
+        "false_hold_rate": 0.2
+      }
+    },
+    "a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
+      "cpc_mc_stereo": {
+        "n": 5,
+        "correct": 3,
+        "tn": 1,
+        "fp": 2,
+        "fn": 0,
+        "tp": 2,
+        "accuracy": 0.6,
+        "false_reply_rate": 0.6666666666666666,
+        "false_hold_rate": 0.0
+      },
+      "cpc_mc_user_only": {
+        "n": 5,
+        "correct": 3,
+        "tn": 1,
+        "fp": 2,
+        "fn": 0,
+        "tp": 2,
+        "accuracy": 0.6,
+        "false_reply_rate": 0.6666666666666666,
+        "false_hold_rate": 0.0
+      },
+      "mimi_standard": {
+        "n": 5,
+        "correct": 2,
+        "tn": 1,
+        "fp": 2,
+        "fn": 1,
+        "tp": 1,
+        "accuracy": 0.4,
+        "false_reply_rate": 0.6666666666666666,
+        "false_hold_rate": 0.5
+      },
+      "mimi_mc": {
+        "n": 5,
+        "correct": 3,
+        "tn": 1,
+        "fp": 2,
+        "fn": 0,
+        "tp": 2,
+        "accuracy": 0.6,
+        "false_reply_rate": 0.6666666666666666,
+        "false_hold_rate": 0.0
+      }
+    },
+    "cb3238dc-3946-42a5-aad8-9b9a54226363": {
+      "cpc_mc_stereo": {
+        "n": 9,
+        "correct": 8,
+        "tn": 5,
+        "fp": 1,
+        "fn": 0,
+        "tp": 3,
+        "accuracy": 0.8888888888888888,
+        "false_reply_rate": 0.16666666666666666,
+        "false_hold_rate": 0.0
+      },
+      "cpc_mc_user_only": {
+        "n": 9,
+        "correct": 4,
+        "tn": 1,
+        "fp": 5,
+        "fn": 0,
+        "tp": 3,
+        "accuracy": 0.4444444444444444,
+        "false_reply_rate": 0.8333333333333334,
+        "false_hold_rate": 0.0
+      },
+      "mimi_standard": {
+        "n": 9,
+        "correct": 7,
+        "tn": 4,
+        "fp": 2,
+        "fn": 0,
+        "tp": 3,
+        "accuracy": 0.7777777777777778,
+        "false_reply_rate": 0.3333333333333333,
+        "false_hold_rate": 0.0
+      },
+      "mimi_mc": {
+        "n": 9,
+        "correct": 7,
+        "tn": 4,
+        "fp": 2,
+        "fn": 0,
+        "tp": 3,
+        "accuracy": 0.7777777777777778,
+        "false_reply_rate": 0.3333333333333333,
+        "false_hold_rate": 0.0
+      }
+    },
+    "e304f740-85d4-427e-9fd6-c5474e9cd737": {
+      "cpc_mc_stereo": {
+        "n": 7,
+        "correct": 6,
+        "tn": 3,
+        "fp": 0,
+        "fn": 1,
+        "tp": 3,
+        "accuracy": 0.8571428571428571,
+        "false_reply_rate": 0.0,
+        "false_hold_rate": 0.25
+      },
+      "cpc_mc_user_only": {
+        "n": 7,
+        "correct": 4,
+        "tn": 0,
+        "fp": 3,
+        "fn": 0,
+        "tp": 4,
+        "accuracy": 0.5714285714285714,
+        "false_reply_rate": 1.0,
+        "false_hold_rate": 0.0
+      },
+      "mimi_standard": {
+        "n": 7,
+        "correct": 7,
+        "tn": 3,
+        "fp": 0,
+        "fn": 0,
+        "tp": 4,
+        "accuracy": 1.0,
+        "false_reply_rate": 0.0,
+        "false_hold_rate": 0.0
+      },
+      "mimi_mc": {
+        "n": 7,
+        "correct": 7,
+        "tn": 3,
+        "fp": 0,
+        "fn": 0,
+        "tp": 4,
+        "accuracy": 1.0,
+        "false_reply_rate": 0.0,
+        "false_hold_rate": 0.0
+      }
+    }
+  },
+  "sources": {
+    "mimi-standard": {
+      "summary_sha256": "b2524af78cd9768a6ee4f3c73e48a6bfc8f5482274acb503bdb0881f70a2ab29",
+      "predictions_sha256": "1f10ead96648b26ec5bea0cee0ed5dfd7c357886b162a44e8b46f4648f4dfcf8"
+    },
+    "mimi-mc": {
+      "summary_sha256": "f7f87c19dfd894771896ce798683b7d2f18bcf0bc2e13da185b9f6c71646359f",
+      "predictions_sha256": "80d850b0044e365cf4554e116f3617af72c7e4a56eb44aaede00004856c20a2b"
+    },
+    "user-only": {
+      "summary_sha256": "99577db77517e31663256c2489ff2300883485ff85a1d04608e467a0bf7e6f9a",
+      "predictions_sha256": "266f43c91d232e5fa7e8e9fc10c294c8f7e7df64fac2332a6a7441faf15d30a8"
+    }
+  },
+  "label_sha256": "780e50f976daf039ab976ea155bdb5d1ca5fffc792f9db7adf8063ec3048ff8e",
+  "audio_sha256": {
+    "2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
+    "40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
+    "872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
+    "a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
+    "cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
+    "e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
+  },
+  "limitations": [
+    "Single seed 42 with exact-repeat witness, not multiple-seed statistics.",
+    "Only 44 selected points in six calls; descriptive comparison, not a held-out generalization claim.",
+    "Mimi standard/MC share the same encoder and framing; CPC uses a different encoder and native 100ms framing.",
+    "All scores are activity-projection mappings; no endpointing-policy or full-dialogue cutoff/latency claims."
+  ]
+}
@@ -0,0 +1,13 @@
+MaAI Mimi 与 CPC 对比（相同当前 44 点，阈值严格 > 0.5）
+
+模型 | 正确数 | 准确率 | 误放行 FP/22 | 误阻止 FN/22 | 相对 CPC 双声道准确率变化
+CPC 降噪版（双声道） | 32/44 | 72.73% | 8/22 | 4/22 | +0.00 个百分点（相对 +0.00%）
+CPC 降噪版（机器人静音） | 23/44 | 52.27% | 18/22 | 3/22 | -20.45 个百分点（相对 -28.13%）
+Mimi 标准版（双声道） | 30/44 | 68.18% | 7/22 | 7/22 | -4.55 个百分点（相对 -6.25%）
+Mimi 降噪版（双声道） | 34/44 | 77.27% | 7/22 | 3/22 | +4.55 个百分点（相对 +6.25%）
+
+1. 标准版与降噪版的结果差异可直接从上表查看；二者使用相同 Mimi 编码器、双声道输入及 80 ms 帧，变化来自所选 VAP 权重。
+2. 与 CPC 比较时同时存在编码器、checkpoint 和原生帧率差异；不把结果归结为 Mimi 编码器的单独因果效果。
+3. 固定种子重复结果一致，但没有多种子误差条；六通录音的 44 个选定点不足以证明泛化优劣。
+
+建议下一步：扩充并核验按通话隔离的测试集，在独立验证集上选择阈值；若要单独研究编码器，补充标准/降噪与 CPC/Mimi 的完整交叉配置。