772 lines
23 KiB
JSON
772 lines
23 KiB
JSON
{
|
|
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
|
"metrics": {
|
|
"soulx_05": {
|
|
"n": 44,
|
|
"tn": 1,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.9545454545454546,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"soulx_08": {
|
|
"n": 44,
|
|
"tn": 2,
|
|
"fp": 20,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.9090909090909091,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"soulx_native_complete": {
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546
|
|
},
|
|
"soulx_corrected_complete": {
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546
|
|
},
|
|
"soulx_state_can_reply": {
|
|
"n": 44,
|
|
"tn": 8,
|
|
"fp": 14,
|
|
"fn": 4,
|
|
"tp": 18,
|
|
"accuracy": 0.5909090909090909,
|
|
"false_reply_rate": 0.6363636363636364,
|
|
"false_hold_rate": 0.18181818181818182
|
|
},
|
|
"maai_05": {
|
|
"n": 44,
|
|
"tn": 15,
|
|
"fp": 7,
|
|
"fn": 4,
|
|
"tp": 18,
|
|
"accuracy": 0.75,
|
|
"false_reply_rate": 0.3181818181818182,
|
|
"false_hold_rate": 0.18181818181818182
|
|
},
|
|
"maai_08": {
|
|
"n": 44,
|
|
"tn": 18,
|
|
"fp": 4,
|
|
"fn": 12,
|
|
"tp": 10,
|
|
"accuracy": 0.6363636363636364,
|
|
"false_reply_rate": 0.18181818181818182,
|
|
"false_hold_rate": 0.5454545454545454
|
|
},
|
|
"maai_zero_false_hold": {
|
|
"n": 44,
|
|
"tn": 5,
|
|
"fp": 17,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.6136363636363636,
|
|
"false_reply_rate": 0.7727272727272727,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"smart_05": {
|
|
"n": 44,
|
|
"tn": 4,
|
|
"fp": 18,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5909090909090909,
|
|
"false_reply_rate": 0.8181818181818182,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"smart_08": {
|
|
"n": 44,
|
|
"tn": 6,
|
|
"fp": 16,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.6363636363636364,
|
|
"false_reply_rate": 0.7272727272727273,
|
|
"false_hold_rate": 0.0
|
|
}
|
|
},
|
|
"changed_vs_baseline": {
|
|
"maai_05": {
|
|
"fixed": 4,
|
|
"regressed": 14
|
|
},
|
|
"smart_05": {
|
|
"fixed": 1,
|
|
"regressed": 4
|
|
},
|
|
"smart_08": {
|
|
"fixed": 0,
|
|
"regressed": 5
|
|
}
|
|
},
|
|
"maai_zero_false_hold_scheme": {
|
|
"rule": "1 - p_now[user] > 0.11",
|
|
"threshold": 0.11,
|
|
"metrics": {
|
|
"n": 44,
|
|
"tn": 5,
|
|
"fp": 17,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.6136363636363636,
|
|
"false_reply_rate": 0.7727272727272727,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
|
"selection": "Supplementary operating point selected on these same 44 labels; not an independent validation or a guarantee of zero false holds on new audio.",
|
|
"lowest_positive": {
|
|
"call_id": "2c83253a-7977-4e3c-a387-0aa1ea06fbe1",
|
|
"decision_ms": "88482",
|
|
"score": 0.11465108394622803,
|
|
"text": "尽快，已经影响行车安全了。"
|
|
},
|
|
"model_inference_unchanged": true,
|
|
"same_predictions_using_p_now_robot": true
|
|
},
|
|
"repeated_point_check": {
|
|
"key": [
|
|
"872a1e67-873a-4c49-a598-b38a670afcad",
|
|
"17153"
|
|
],
|
|
"probability_difference": 0.0,
|
|
"same_asr": true,
|
|
"same_raw_state": true
|
|
},
|
|
"native_can_reply_mapping": "Supplementary post-hoc mapping: start hold; complete -> allow; incomplete/nonidle -> hold; idle/backchannel retain previous value. No timeout, no service resets, no external EOT signals, no future frames.",
|
|
"native_can_reply_changed_vs_baseline": {
|
|
"maai_05": {
|
|
"fixed": 5,
|
|
"regressed": 12
|
|
},
|
|
"smart_05": {
|
|
"fixed": 6,
|
|
"regressed": 6
|
|
},
|
|
"smart_08": {
|
|
"fixed": 4,
|
|
"regressed": 6
|
|
}
|
|
},
|
|
"soulx_native_state_metrics": {
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546
|
|
},
|
|
"soulx_corrected_state_metrics": {
|
|
"n": 44,
|
|
"tn": 22,
|
|
"fp": 0,
|
|
"fn": 21,
|
|
"tp": 1,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.0,
|
|
"false_hold_rate": 0.9545454545454546
|
|
},
|
|
"soulx_threshold_diagnostics": [
|
|
{
|
|
"threshold": 0.1,
|
|
"n": 44,
|
|
"tn": 0,
|
|
"fp": 22,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.2,
|
|
"n": 44,
|
|
"tn": 0,
|
|
"fp": 22,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.3,
|
|
"n": 44,
|
|
"tn": 0,
|
|
"fp": 22,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.4,
|
|
"n": 44,
|
|
"tn": 0,
|
|
"fp": 22,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 1.0,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.5,
|
|
"n": 44,
|
|
"tn": 1,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.9545454545454546,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.6,
|
|
"n": 44,
|
|
"tn": 1,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.9545454545454546,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.7,
|
|
"n": 44,
|
|
"tn": 1,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.9545454545454546,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.8,
|
|
"n": 44,
|
|
"tn": 2,
|
|
"fp": 20,
|
|
"fn": 0,
|
|
"tp": 22,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.9090909090909091,
|
|
"false_hold_rate": 0.0
|
|
},
|
|
{
|
|
"threshold": 0.9,
|
|
"n": 44,
|
|
"tn": 5,
|
|
"fp": 17,
|
|
"fn": 3,
|
|
"tp": 19,
|
|
"accuracy": 0.5454545454545454,
|
|
"false_reply_rate": 0.7727272727272727,
|
|
"false_hold_rate": 0.13636363636363635
|
|
},
|
|
{
|
|
"threshold": 0.95,
|
|
"n": 44,
|
|
"tn": 12,
|
|
"fp": 10,
|
|
"fn": 12,
|
|
"tp": 10,
|
|
"accuracy": 0.5,
|
|
"false_reply_rate": 0.45454545454545453,
|
|
"false_hold_rate": 0.5454545454545454
|
|
},
|
|
{
|
|
"threshold": 0.99,
|
|
"n": 44,
|
|
"tn": 21,
|
|
"fp": 1,
|
|
"fn": 20,
|
|
"tp": 2,
|
|
"accuracy": 0.5227272727272727,
|
|
"false_reply_rate": 0.045454545454545456,
|
|
"false_hold_rate": 0.9090909090909091
|
|
}
|
|
],
|
|
"latency": {
|
|
"soulx": {
|
|
"metadata": {
|
|
"host": "2033lab04ztr",
|
|
"hostname": "ubuntu",
|
|
"python": "3.10.22 (main, Oct 1 2026, 20:14:04) [GCC 14.3.0]",
|
|
"started_at_utc": "2026-10-13T15:42:48.661080+00:00",
|
|
"gpu_driver": "595.84",
|
|
"gpu": "NVIDIA GeForce RTX 3090",
|
|
"cpu_threads": 4,
|
|
"torch": "2.6.0+cu124",
|
|
"cuda": "12.4",
|
|
"load_ms": 58031.039100999806,
|
|
"parameter_dtypes": [
|
|
"torch.float32"
|
|
],
|
|
"declared_precision": "bf16",
|
|
"actual_precision_note": "Official loader does not cast to bf16; no autocast added.",
|
|
"checkpoint_loading": [
|
|
{
|
|
"missing": [],
|
|
"unexpected": [
|
|
"embed_tokens_func.weight"
|
|
]
|
|
}
|
|
],
|
|
"repo_commit": "a0b9063843df69619b087b95b74597b2176910b8",
|
|
"model_revision": "61701c4ab8193cc1ee2220d3848872ac6c720142",
|
|
"model_checksums": {
|
|
"repo": "Soul-AILab/SoulX-Duplug-0.6B",
|
|
"revision": "61701c4ab8193cc1ee2220d3848872ac6c720142",
|
|
"files": [
|
|
{
|
|
"file": ".gitattributes",
|
|
"bytes": 1655,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/added_tokens.json",
|
|
"bytes": 1494656,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/chat_template.jinja",
|
|
"bytes": 4168,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/config.json",
|
|
"bytes": 726,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/config_record.yaml",
|
|
"bytes": 172,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/generation_config.json",
|
|
"bytes": 214,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/merges.txt",
|
|
"bytes": 1671853,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/model.safetensors",
|
|
"bytes": 2384234968,
|
|
"lfs_sha256": "82f718371044abd7d05e4de2dbfd766938eb9d07a0c7152e8e4bb20cab53b6ec"
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/special_tokens_map.json",
|
|
"bytes": 613,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/tokenizer.json",
|
|
"bytes": 21375814,
|
|
"lfs_sha256": "9188dc1b4ec9935bbc52b70fd43e7e39718e44282b937055918e58ff5234e975"
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/tokenizer_config.json",
|
|
"bytes": 9439594,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "Qwen3-0.6B-expand_vocab_v2/vocab.json",
|
|
"bytes": 2776833,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "README.md",
|
|
"bytes": 6555,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "SoulX-Duplug-logo.png",
|
|
"bytes": 763847,
|
|
"lfs_sha256": "5ae357ebc267373d75560f127817d8e7e381215ac2b6d70663c7ff98de6bfe58"
|
|
},
|
|
{
|
|
"file": "SoulX-Duplug/SoulX-Duplug-0.6B-Bilingual.pth",
|
|
"bytes": 3896904668,
|
|
"lfs_sha256": "b0703dea0b1dbb1cd51e6e7b6514c60907ea4d4b6752cecc3f71cb6445650dbe"
|
|
},
|
|
{
|
|
"file": "config.yaml",
|
|
"bytes": 246,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/.msc",
|
|
"bytes": 440,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/.mv",
|
|
"bytes": 36,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/LICENSE",
|
|
"bytes": 6490,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/README.md",
|
|
"bytes": 1296,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/config.json",
|
|
"bytes": 1771,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/configuration.json",
|
|
"bytes": 57,
|
|
"lfs_sha256": null
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/model.safetensors",
|
|
"bytes": 1458374480,
|
|
"lfs_sha256": "2800bd503f52b51e45f0c53cfd5c31dcfe8ef7f13d22b396aa3d53e0280dd1e4"
|
|
},
|
|
{
|
|
"file": "glm-4-voice-tokenizer/preprocessor_config.json",
|
|
"bytes": 340,
|
|
"lfs_sha256": null
|
|
}
|
|
]
|
|
},
|
|
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
|
"script_sha256": "8917b789e7fb2753ba621f4e50c1aa110e4db5b03fd205b6215efa3def8667c0",
|
|
"dependencies": {
|
|
"transformers": "4.52.1",
|
|
"funasr": "1.2.6",
|
|
"modelscope": "1.28.2",
|
|
"peft": "0.16.0",
|
|
"pytorch-lightning": "2.5.2",
|
|
"numpy": "1.26.4",
|
|
"soxr": "0.5.0.post1"
|
|
}
|
|
},
|
|
"complete": true,
|
|
"update_interval_ms": 160,
|
|
"lookahead_ms": 40,
|
|
"summary": {
|
|
"all": {
|
|
"total_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 115.09665274500838,
|
|
"median_ms": 91.10886999997092,
|
|
"p95_ms": 209.43407925005886,
|
|
"p99_ms": 236.85046370064805,
|
|
"min_ms": 86.25251000012213,
|
|
"max_ms": 250.48965200039675
|
|
},
|
|
"audio_tokens_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 12.331983890980364,
|
|
"median_ms": 12.281107499802602,
|
|
"p95_ms": 12.911304250565081,
|
|
"p99_ms": 13.458867049848775,
|
|
"min_ms": 11.758304000068165,
|
|
"max_ms": 14.449737999711942
|
|
},
|
|
"asr_stage_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 63.29801463820195,
|
|
"median_ms": 38.69248599994535,
|
|
"p95_ms": 159.31682424957216,
|
|
"p99_ms": 186.5517717498278,
|
|
"min_ms": 35.78745800041361,
|
|
"max_ms": 199.4932890002019
|
|
},
|
|
"state_model_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 37.18374523282112,
|
|
"median_ms": 36.97866250013249,
|
|
"p95_ms": 38.905622500351456,
|
|
"p99_ms": 39.68806565048907,
|
|
"min_ms": 35.525741999663296,
|
|
"max_ms": 41.528222000124515
|
|
},
|
|
"cascade_asr_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 23.16353569586444,
|
|
"median_ms": 0.0,
|
|
"p95_ms": 111.09653949984022,
|
|
"p99_ms": 120.70468979977701,
|
|
"min_ms": 0.0,
|
|
"max_ms": 124.45981700057018
|
|
},
|
|
"non_cascade_ms": {
|
|
"n": 2706,
|
|
"mean_ms": 91.93311704914395,
|
|
"median_ms": 90.45315400044274,
|
|
"p95_ms": 93.65244125024219,
|
|
"p99_ms": 129.31540989952737,
|
|
"min_ms": 85.55587400041986,
|
|
"max_ms": 131.7838019986084
|
|
}
|
|
},
|
|
"steady": {
|
|
"total_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 117.67224802351365,
|
|
"median_ms": 91.21004799999355,
|
|
"p95_ms": 211.0119177505112,
|
|
"p99_ms": 238.42288805012686,
|
|
"min_ms": 87.33008800027164,
|
|
"max_ms": 250.48965200039675
|
|
},
|
|
"audio_tokens_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 12.273323957560097,
|
|
"median_ms": 12.248947999523807,
|
|
"p95_ms": 12.626806250182199,
|
|
"p99_ms": 12.966947899940351,
|
|
"min_ms": 11.758304000068165,
|
|
"max_ms": 13.877598000362923
|
|
},
|
|
"asr_stage_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 65.82940533740637,
|
|
"median_ms": 38.71273200002179,
|
|
"p95_ms": 160.64461450037015,
|
|
"p99_ms": 187.41259069984156,
|
|
"min_ms": 36.664592000306584,
|
|
"max_ms": 199.4932890002019
|
|
},
|
|
"state_model_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 37.24377172341971,
|
|
"median_ms": 37.03352750017075,
|
|
"p95_ms": 39.0115195002636,
|
|
"p99_ms": 39.779913449910964,
|
|
"min_ms": 35.525741999663296,
|
|
"max_ms": 41.528222000124515
|
|
},
|
|
"cascade_asr_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 25.609160123213883,
|
|
"median_ms": 0.0,
|
|
"p95_ms": 113.19229975015332,
|
|
"p99_ms": 121.38709505015868,
|
|
"min_ms": 0.0,
|
|
"max_ms": 124.45981700057018
|
|
},
|
|
"non_cascade_ms": {
|
|
"n": 1956,
|
|
"mean_ms": 92.06308790029976,
|
|
"median_ms": 90.51483399980498,
|
|
"p95_ms": 93.72818474980704,
|
|
"p99_ms": 129.4210380507593,
|
|
"min_ms": 85.55587400041986,
|
|
"max_ms": 131.7838019986084
|
|
}
|
|
},
|
|
"asr_called": {
|
|
"total_ms": {
|
|
"n": 642,
|
|
"mean_ms": 193.67558148909285,
|
|
"median_ms": 196.2352535001628,
|
|
"p95_ms": 235.79334210030538,
|
|
"p99_ms": 244.87408848051928,
|
|
"min_ms": 100.09163700033241,
|
|
"max_ms": 250.48965200039675
|
|
},
|
|
"audio_tokens_ms": {
|
|
"n": 642,
|
|
"mean_ms": 12.260155926785366,
|
|
"median_ms": 12.208873500185291,
|
|
"p95_ms": 12.747471649481668,
|
|
"p99_ms": 13.129559900653478,
|
|
"min_ms": 11.764587000470783,
|
|
"max_ms": 13.877598000362923
|
|
},
|
|
"asr_stage_ms": {
|
|
"n": 642,
|
|
"mean_ms": 142.97833891432276,
|
|
"median_ms": 145.64249349950842,
|
|
"p95_ms": 185.2425876997586,
|
|
"p99_ms": 193.64899680030248,
|
|
"min_ms": 50.900377000289154,
|
|
"max_ms": 199.4932890002019
|
|
},
|
|
"state_model_ms": {
|
|
"n": 642,
|
|
"mean_ms": 37.97521542678468,
|
|
"median_ms": 37.99808250005299,
|
|
"p95_ms": 39.627456049856846,
|
|
"p99_ms": 40.04239527022946,
|
|
"min_ms": 35.525741999663296,
|
|
"max_ms": 41.528222000124515
|
|
},
|
|
"cascade_asr_ms": {
|
|
"n": 642,
|
|
"mean_ms": 97.63322054985852,
|
|
"median_ms": 103.59336999999869,
|
|
"p95_ms": 120.26240784971378,
|
|
"p99_ms": 123.0363232399759,
|
|
"min_ms": 12.641286999496515,
|
|
"max_ms": 124.45981700057018
|
|
},
|
|
"non_cascade_ms": {
|
|
"n": 642,
|
|
"mean_ms": 96.04236093923433,
|
|
"median_ms": 89.76878849989589,
|
|
"p95_ms": 128.9486389001013,
|
|
"p99_ms": 130.96938659011357,
|
|
"min_ms": 85.55587400041986,
|
|
"max_ms": 131.7838019986084
|
|
}
|
|
},
|
|
"no_asr": {
|
|
"total_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 90.65495107170302,
|
|
"median_ms": 90.5710160000126,
|
|
"p95_ms": 92.65106535003724,
|
|
"p99_ms": 93.6572591400909,
|
|
"min_ms": 86.25251000012213,
|
|
"max_ms": 103.34190300000046
|
|
},
|
|
"audio_tokens_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 12.354325728680553,
|
|
"median_ms": 12.30188099998486,
|
|
"p95_ms": 12.953575500205261,
|
|
"p99_ms": 13.518332969752008,
|
|
"min_ms": 11.758304000068165,
|
|
"max_ms": 14.449737999711942
|
|
},
|
|
"asr_stage_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 38.51372772673414,
|
|
"median_ms": 38.45273399974758,
|
|
"p95_ms": 39.44197145033286,
|
|
"p99_ms": 39.8342825799682,
|
|
"min_ms": 35.78745800041361,
|
|
"max_ms": 40.66350199991575
|
|
},
|
|
"state_model_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 36.937561189931294,
|
|
"median_ms": 36.847997499990015,
|
|
"p95_ms": 37.8194476995759,
|
|
"p99_ms": 38.303353680221335,
|
|
"min_ms": 35.57524700045178,
|
|
"max_ms": 39.21048299980612
|
|
},
|
|
"cascade_asr_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 0.0,
|
|
"median_ms": 0.0,
|
|
"p95_ms": 0.0,
|
|
"p99_ms": 0.0,
|
|
"min_ms": 0.0,
|
|
"max_ms": 0.0
|
|
},
|
|
"non_cascade_ms": {
|
|
"n": 2064,
|
|
"mean_ms": 90.65495107170302,
|
|
"median_ms": 90.5710160000126,
|
|
"p95_ms": 92.65106535003724,
|
|
"p99_ms": 93.6572591400909,
|
|
"min_ms": 86.25251000012213,
|
|
"max_ms": 103.34190300000046
|
|
}
|
|
}
|
|
},
|
|
"realtime_factor": 0.7193540796563035,
|
|
"fraction_over_160ms": 0.21840354767184036,
|
|
"peak_gpu_allocated_bytes": 6640691712,
|
|
"peak_gpu_reserved_bytes": 6866075648,
|
|
"note": "CUDA synchronized wall time. Includes waveform features, audio tokenizer, LLM state, cascade ASR and instrumentation. Excludes imports/model loading, file decode/resample, network, reply policy. Official FP32 loading retained; BF16 config does not cast weights."
|
|
},
|
|
"local_baselines": {
|
|
"date": "2026-10-13",
|
|
"host": "Apple M4 Mac mini, 16 GB, CPU only",
|
|
"smart_turn_1_thread": {
|
|
"n": 220,
|
|
"mean_ms": 31.537017971277237,
|
|
"median_ms": 31.260979129001498,
|
|
"p95_ms": 33.23495052754878,
|
|
"p99_ms": 37.6575198629871,
|
|
"min_ms": 30.356083996593952,
|
|
"max_ms": 39.30716682225466
|
|
},
|
|
"smart_turn_4_threads": {
|
|
"n": 220,
|
|
"mean_ms": 11.686938256025314,
|
|
"median_ms": 11.510478798300028,
|
|
"p95_ms": 12.558008311316371,
|
|
"p99_ms": 13.575859060510993,
|
|
"min_ms": 11.002332903444767,
|
|
"max_ms": 19.810708239674568
|
|
},
|
|
"maai_4_threads_incremental": {
|
|
"n": 3133,
|
|
"mean_ms": 4.907698216980357,
|
|
"median_ms": 4.787249956279993,
|
|
"p95_ms": 5.38598345592618,
|
|
"p99_ms": 7.23788535222411,
|
|
"min_ms": 4.287166986614466,
|
|
"max_ms": 24.128540884703398
|
|
},
|
|
"equal_thread_latency_ratio": 2.3813481879527907,
|
|
"metadata": {
|
|
"smart_turn_includes_resample": true,
|
|
"smart_turn_resample_mean_ms": 0.16240161547267978,
|
|
"maai_input_sample_rate": 16000,
|
|
"maai_update_interval_ms": 100,
|
|
"maai_cache_seconds": 20,
|
|
"smart_turn_window_seconds": 8,
|
|
"maai_offline_replay_mean_ms": 976.480005774647,
|
|
"excludes": [
|
|
"model import/loading",
|
|
"file decoding",
|
|
"VAD silence wait",
|
|
"ASR/LLM/TTS/network waits"
|
|
]
|
|
},
|
|
"source_files": [
|
|
"experiments/smart-turn-v3.2/results/latency.json",
|
|
"experiments/maai-ch-kyoto/results/latency.json"
|
|
]
|
|
}
|
|
},
|
|
"method": "Each point resets native KV state, replays <=20s past user audio with built-in Paraformer. Final process chunk ends D-40ms and lookahead ends D. No label transcript input, no future audio, no fallback reply or complete-trigger reset.",
|
|
"limitations": [
|
|
"Conditional two-token score is not a calibrated can_reply probability; raw five-state argmax retained as a diagnostic.",
|
|
"29 draft labels and 15 labels without verification metadata; selected-point exploratory evaluation.",
|
|
"Input channels/context and hardware differ across models; this is a configured-system comparison.",
|
|
"Independent point restart differs from continuous server state; ASR is part of SoulX inference.",
|
|
"Known ASR/label alignment issues at cb3238dc 13540 and e304f740 11531 remain unchanged."
|
|
]
|
|
}
|