330 lines
8.6 KiB
JSON
330 lines
8.6 KiB
JSON
|
{
|
||
|
|
"total_points": 44,
|
||
|
|
"metrics": {
|
||
|
|
"n": 44,
|
||
|
|
"tn": 4,
|
||
|
|
"fp": 18,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 22,
|
||
|
|
"accuracy": 0.5909090909090909,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.55,
|
||
|
|
"f1": 0.7096774193548387
|
||
|
|
},
|
||
|
|
"grouped": {
|
||
|
|
"label_status": {
|
||
|
|
"draft": {
|
||
|
|
"n": 29,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 9,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 18,
|
||
|
|
"accuracy": 0.6896551724137931,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.6666666666666666,
|
||
|
|
"f1": 0.8
|
||
|
|
},
|
||
|
|
"unconfirmed_manual": {
|
||
|
|
"n": 15,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 9,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.4,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.3076923076923077,
|
||
|
|
"f1": 0.47058823529411764
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"call_id": {
|
||
|
|
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||
|
|
"n": 6,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 2,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.6666666666666666,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.6666666666666666,
|
||
|
|
"f1": 0.8
|
||
|
|
},
|
||
|
|
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||
|
|
"n": 8,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 4,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5,
|
||
|
|
"f1": 0.6666666666666666
|
||
|
|
},
|
||
|
|
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||
|
|
"n": 9,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 4,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 5,
|
||
|
|
"accuracy": 0.5555555555555556,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5555555555555556,
|
||
|
|
"f1": 0.7142857142857143
|
||
|
|
},
|
||
|
|
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||
|
|
"n": 5,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 3,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 2,
|
||
|
|
"accuracy": 0.4,
|
||
|
|
"false_reply_rate": 1.0,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.4,
|
||
|
|
"f1": 0.5714285714285714
|
||
|
|
},
|
||
|
|
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||
|
|
"n": 9,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 4,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 3,
|
||
|
|
"accuracy": 0.5555555555555556,
|
||
|
|
"false_reply_rate": 0.6666666666666666,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.42857142857142855,
|
||
|
|
"f1": 0.6
|
||
|
|
},
|
||
|
|
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||
|
|
"n": 7,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 1,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 0.8571428571428571,
|
||
|
|
"false_reply_rate": 0.3333333333333333,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.8,
|
||
|
|
"f1": 0.8888888888888888
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"reason": {
|
||
|
|
"completed_answer": {
|
||
|
|
"n": 18,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 18,
|
||
|
|
"accuracy": 1.0,
|
||
|
|
"false_reply_rate": null,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 1.0,
|
||
|
|
"f1": 1.0
|
||
|
|
},
|
||
|
|
"completed_turn": {
|
||
|
|
"n": 4,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 4,
|
||
|
|
"accuracy": 1.0,
|
||
|
|
"false_reply_rate": null,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 1.0,
|
||
|
|
"f1": 1.0
|
||
|
|
},
|
||
|
|
"mid_turn_pause": {
|
||
|
|
"n": 17,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 15,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 0,
|
||
|
|
"accuracy": 0.11764705882352941,
|
||
|
|
"false_reply_rate": 0.8823529411764706,
|
||
|
|
"false_hold_rate": null,
|
||
|
|
"recall": null,
|
||
|
|
"precision": 0.0,
|
||
|
|
"f1": 0.0
|
||
|
|
},
|
||
|
|
"unfinished_expression": {
|
||
|
|
"n": 5,
|
||
|
|
"tn": 2,
|
||
|
|
"fp": 3,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 0,
|
||
|
|
"accuracy": 0.4,
|
||
|
|
"false_reply_rate": 0.6,
|
||
|
|
"false_hold_rate": null,
|
||
|
|
"recall": null,
|
||
|
|
"precision": 0.0,
|
||
|
|
"f1": 0.0
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"input_source": {
|
||
|
|
"asr_user_block": {
|
||
|
|
"n": 42,
|
||
|
|
"tn": 4,
|
||
|
|
"fp": 18,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 20,
|
||
|
|
"accuracy": 0.5714285714285714,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5263157894736842,
|
||
|
|
"f1": 0.6896551724137931
|
||
|
|
},
|
||
|
|
"past_vad_missing_asr": {
|
||
|
|
"n": 2,
|
||
|
|
"tn": 0,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 2,
|
||
|
|
"accuracy": 1.0,
|
||
|
|
"false_reply_rate": null,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 1.0,
|
||
|
|
"f1": 1.0
|
||
|
|
}
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"config": {
|
||
|
|
"pipecat_version": "1.12.0",
|
||
|
|
"model": "smart-turn-v3.2-cpu.onnx",
|
||
|
|
"threshold": 0.5,
|
||
|
|
"pre_speech_ms": 500,
|
||
|
|
"max_duration_secs": 8,
|
||
|
|
"continuation_gap_ms": 3000,
|
||
|
|
"selected_audio_channel": 0,
|
||
|
|
"model_sha256": "2bb026316b14a660486a75b1733cd3fbab8c2fd0314dc9af7be49f8cca967e4f",
|
||
|
|
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||
|
|
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa"
|
||
|
|
},
|
||
|
|
"method": "Each exact decision_ms is classified directly by the model. Earlier model scores, EOT signals, VAD stop gates, silence timeout forced replies and ASR return gates never determine prediction or clear this input.",
|
||
|
|
"threshold_diagnostics": [
|
||
|
|
{
|
||
|
|
"threshold": 0.5,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 4,
|
||
|
|
"fp": 18,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 22,
|
||
|
|
"accuracy": 0.5909090909090909,
|
||
|
|
"false_reply_rate": 0.8181818181818182,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.55,
|
||
|
|
"f1": 0.7096774193548387
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.7,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 5,
|
||
|
|
"fp": 17,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 22,
|
||
|
|
"accuracy": 0.6136363636363636,
|
||
|
|
"false_reply_rate": 0.7727272727272727,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5641025641025641,
|
||
|
|
"f1": 0.7213114754098361
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.8,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 6,
|
||
|
|
"fp": 16,
|
||
|
|
"fn": 0,
|
||
|
|
"tp": 22,
|
||
|
|
"accuracy": 0.6363636363636364,
|
||
|
|
"false_reply_rate": 0.7272727272727273,
|
||
|
|
"false_hold_rate": 0.0,
|
||
|
|
"recall": 1.0,
|
||
|
|
"precision": 0.5789473684210527,
|
||
|
|
"f1": 0.7333333333333333
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.9,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 7,
|
||
|
|
"fp": 15,
|
||
|
|
"fn": 1,
|
||
|
|
"tp": 21,
|
||
|
|
"accuracy": 0.6363636363636364,
|
||
|
|
"false_reply_rate": 0.6818181818181818,
|
||
|
|
"false_hold_rate": 0.045454545454545456,
|
||
|
|
"recall": 0.9545454545454546,
|
||
|
|
"precision": 0.5833333333333334,
|
||
|
|
"f1": 0.7241379310344828
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.95,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 9,
|
||
|
|
"fp": 13,
|
||
|
|
"fn": 3,
|
||
|
|
"tp": 19,
|
||
|
|
"accuracy": 0.6363636363636364,
|
||
|
|
"false_reply_rate": 0.5909090909090909,
|
||
|
|
"false_hold_rate": 0.13636363636363635,
|
||
|
|
"recall": 0.8636363636363636,
|
||
|
|
"precision": 0.59375,
|
||
|
|
"f1": 0.7037037037037037
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.98,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 14,
|
||
|
|
"fp": 8,
|
||
|
|
"fn": 6,
|
||
|
|
"tp": 16,
|
||
|
|
"accuracy": 0.6818181818181818,
|
||
|
|
"false_reply_rate": 0.36363636363636365,
|
||
|
|
"false_hold_rate": 0.2727272727272727,
|
||
|
|
"recall": 0.7272727272727273,
|
||
|
|
"precision": 0.6666666666666666,
|
||
|
|
"f1": 0.6956521739130435
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"threshold": 0.99,
|
||
|
|
"n": 44,
|
||
|
|
"tn": 22,
|
||
|
|
"fp": 0,
|
||
|
|
"fn": 22,
|
||
|
|
"tp": 0,
|
||
|
|
"accuracy": 0.5,
|
||
|
|
"false_reply_rate": 0.0,
|
||
|
|
"false_hold_rate": 1.0,
|
||
|
|
"recall": 0.0,
|
||
|
|
"precision": null,
|
||
|
|
"f1": 0.0
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"audio_origin": "Past ASR user segments locate current speech block. Consecutive user segments within 3 s share the first start unless a complete non-overlapping robot question separates them. Keep 500 ms before that start and then last 8 s. Intra-segment pauses are not resolved by coarse ASR timing. Two missing-ASR points use only past Silero VAD starts; full-audio model events are not used.",
|
||
|
|
"limitations": [
|
||
|
|
"Exploratory scoring: 29 xlsx_draft and 15 rows missing annotator/confidence; no explicit verified status.",
|
||
|
|
"Coarse ASR speech block origin approximates the discussed cache rule; this is point classification, not a complete online Pipecat strategy replay.",
|
||
|
|
"cb3238dc 13540 text is 辽宁葫芦岛 but its corresponding user ASR starts at 16600; review alignment. e304f740 11531 surname utterance is absent from ASR history.",
|
||
|
|
"User channel 0 can contain robot echo/overlap; no future ASR-based masking.",
|
||
|
|
"Selected-point false-reply rate is not interval-level interruption rate."
|
||
|
|
]
|
||
|
|
}
|