590 lines
15 KiB
JSON
590 lines
15 KiB
JSON
{
|
||
"total_points": 44,
|
||
"main_rule": "p_now_robot > 0.5",
|
||
"main_metrics": {
|
||
"n": 44,
|
||
"tn": 15,
|
||
"fp": 7,
|
||
"fn": 4,
|
||
"tp": 18,
|
||
"accuracy": 0.75,
|
||
"false_reply_rate": 0.3181818181818182,
|
||
"false_hold_rate": 0.18181818181818182,
|
||
"recall": 0.8181818181818182,
|
||
"precision": 0.72,
|
||
"f1": 0.7659574468085106
|
||
},
|
||
"rule_metrics": {
|
||
"p_now_robot": {
|
||
"n": 44,
|
||
"tn": 15,
|
||
"fp": 7,
|
||
"fn": 4,
|
||
"tp": 18,
|
||
"accuracy": 0.75,
|
||
"false_reply_rate": 0.3181818181818182,
|
||
"false_hold_rate": 0.18181818181818182,
|
||
"recall": 0.8181818181818182,
|
||
"precision": 0.72,
|
||
"f1": 0.7659574468085106
|
||
},
|
||
"p_future_robot": {
|
||
"n": 44,
|
||
"tn": 13,
|
||
"fp": 9,
|
||
"fn": 5,
|
||
"tp": 17,
|
||
"accuracy": 0.6818181818181818,
|
||
"false_reply_rate": 0.4090909090909091,
|
||
"false_hold_rate": 0.22727272727272727,
|
||
"recall": 0.7727272727272727,
|
||
"precision": 0.6538461538461539,
|
||
"f1": 0.7083333333333334
|
||
},
|
||
"both_robot": {
|
||
"n": 44,
|
||
"tn": 15,
|
||
"fp": 7,
|
||
"fn": 5,
|
||
"tp": 17,
|
||
"accuracy": 0.7272727272727273,
|
||
"false_reply_rate": 0.3181818181818182,
|
||
"false_hold_rate": 0.22727272727272727,
|
||
"recall": 0.7727272727272727,
|
||
"precision": 0.7083333333333334,
|
||
"f1": 0.7391304347826086
|
||
}
|
||
},
|
||
"thresholds": [
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.5,
|
||
"n": 44,
|
||
"tn": 15,
|
||
"fp": 7,
|
||
"fn": 4,
|
||
"tp": 18,
|
||
"accuracy": 0.75,
|
||
"false_reply_rate": 0.3181818181818182,
|
||
"false_hold_rate": 0.18181818181818182,
|
||
"recall": 0.8181818181818182,
|
||
"precision": 0.72,
|
||
"f1": 0.7659574468085106
|
||
},
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.7,
|
||
"n": 44,
|
||
"tn": 16,
|
||
"fp": 6,
|
||
"fn": 10,
|
||
"tp": 12,
|
||
"accuracy": 0.6363636363636364,
|
||
"false_reply_rate": 0.2727272727272727,
|
||
"false_hold_rate": 0.45454545454545453,
|
||
"recall": 0.5454545454545454,
|
||
"precision": 0.6666666666666666,
|
||
"f1": 0.6
|
||
},
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.8,
|
||
"n": 44,
|
||
"tn": 18,
|
||
"fp": 4,
|
||
"fn": 12,
|
||
"tp": 10,
|
||
"accuracy": 0.6363636363636364,
|
||
"false_reply_rate": 0.18181818181818182,
|
||
"false_hold_rate": 0.5454545454545454,
|
||
"recall": 0.45454545454545453,
|
||
"precision": 0.7142857142857143,
|
||
"f1": 0.5555555555555556
|
||
},
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.9,
|
||
"n": 44,
|
||
"tn": 20,
|
||
"fp": 2,
|
||
"fn": 13,
|
||
"tp": 9,
|
||
"accuracy": 0.6590909090909091,
|
||
"false_reply_rate": 0.09090909090909091,
|
||
"false_hold_rate": 0.5909090909090909,
|
||
"recall": 0.4090909090909091,
|
||
"precision": 0.8181818181818182,
|
||
"f1": 0.5454545454545454
|
||
},
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.95,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 17,
|
||
"tp": 5,
|
||
"accuracy": 0.6136363636363636,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 0.7727272727272727,
|
||
"recall": 0.22727272727272727,
|
||
"precision": 1.0,
|
||
"f1": 0.37037037037037035
|
||
},
|
||
{
|
||
"rule": "p_now_robot",
|
||
"threshold": 0.98,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 22,
|
||
"tp": 0,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 1.0,
|
||
"recall": 0.0,
|
||
"precision": null,
|
||
"f1": 0.0
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.5,
|
||
"n": 44,
|
||
"tn": 13,
|
||
"fp": 9,
|
||
"fn": 5,
|
||
"tp": 17,
|
||
"accuracy": 0.6818181818181818,
|
||
"false_reply_rate": 0.4090909090909091,
|
||
"false_hold_rate": 0.22727272727272727,
|
||
"recall": 0.7727272727272727,
|
||
"precision": 0.6538461538461539,
|
||
"f1": 0.7083333333333334
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.7,
|
||
"n": 44,
|
||
"tn": 17,
|
||
"fp": 5,
|
||
"fn": 11,
|
||
"tp": 11,
|
||
"accuracy": 0.6363636363636364,
|
||
"false_reply_rate": 0.22727272727272727,
|
||
"false_hold_rate": 0.5,
|
||
"recall": 0.5,
|
||
"precision": 0.6875,
|
||
"f1": 0.5789473684210527
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.8,
|
||
"n": 44,
|
||
"tn": 19,
|
||
"fp": 3,
|
||
"fn": 14,
|
||
"tp": 8,
|
||
"accuracy": 0.6136363636363636,
|
||
"false_reply_rate": 0.13636363636363635,
|
||
"false_hold_rate": 0.6363636363636364,
|
||
"recall": 0.36363636363636365,
|
||
"precision": 0.7272727272727273,
|
||
"f1": 0.48484848484848486
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.9,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 18,
|
||
"tp": 4,
|
||
"accuracy": 0.5909090909090909,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 0.8181818181818182,
|
||
"recall": 0.18181818181818182,
|
||
"precision": 1.0,
|
||
"f1": 0.3076923076923077
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.95,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 22,
|
||
"tp": 0,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 1.0,
|
||
"recall": 0.0,
|
||
"precision": null,
|
||
"f1": 0.0
|
||
},
|
||
{
|
||
"rule": "p_future_robot",
|
||
"threshold": 0.98,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 22,
|
||
"tp": 0,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 1.0,
|
||
"recall": 0.0,
|
||
"precision": null,
|
||
"f1": 0.0
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.5,
|
||
"n": 44,
|
||
"tn": 15,
|
||
"fp": 7,
|
||
"fn": 5,
|
||
"tp": 17,
|
||
"accuracy": 0.7272727272727273,
|
||
"false_reply_rate": 0.3181818181818182,
|
||
"false_hold_rate": 0.22727272727272727,
|
||
"recall": 0.7727272727272727,
|
||
"precision": 0.7083333333333334,
|
||
"f1": 0.7391304347826086
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.7,
|
||
"n": 44,
|
||
"tn": 17,
|
||
"fp": 5,
|
||
"fn": 11,
|
||
"tp": 11,
|
||
"accuracy": 0.6363636363636364,
|
||
"false_reply_rate": 0.22727272727272727,
|
||
"false_hold_rate": 0.5,
|
||
"recall": 0.5,
|
||
"precision": 0.6875,
|
||
"f1": 0.5789473684210527
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.8,
|
||
"n": 44,
|
||
"tn": 19,
|
||
"fp": 3,
|
||
"fn": 14,
|
||
"tp": 8,
|
||
"accuracy": 0.6136363636363636,
|
||
"false_reply_rate": 0.13636363636363635,
|
||
"false_hold_rate": 0.6363636363636364,
|
||
"recall": 0.36363636363636365,
|
||
"precision": 0.7272727272727273,
|
||
"f1": 0.48484848484848486
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.9,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 18,
|
||
"tp": 4,
|
||
"accuracy": 0.5909090909090909,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 0.8181818181818182,
|
||
"recall": 0.18181818181818182,
|
||
"precision": 1.0,
|
||
"f1": 0.3076923076923077
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.95,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 22,
|
||
"tp": 0,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 1.0,
|
||
"recall": 0.0,
|
||
"precision": null,
|
||
"f1": 0.0
|
||
},
|
||
{
|
||
"rule": "both_robot",
|
||
"threshold": 0.98,
|
||
"n": 44,
|
||
"tn": 22,
|
||
"fp": 0,
|
||
"fn": 22,
|
||
"tp": 0,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 1.0,
|
||
"recall": 0.0,
|
||
"precision": null,
|
||
"f1": 0.0
|
||
}
|
||
],
|
||
"grouped": {
|
||
"call_id": {
|
||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": {
|
||
"n": 6,
|
||
"tn": 0,
|
||
"fp": 2,
|
||
"fn": 1,
|
||
"tp": 3,
|
||
"accuracy": 0.5,
|
||
"false_reply_rate": 1.0,
|
||
"false_hold_rate": 0.25,
|
||
"recall": 0.75,
|
||
"precision": 0.6,
|
||
"f1": 0.6666666666666666
|
||
},
|
||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": {
|
||
"n": 8,
|
||
"tn": 3,
|
||
"fp": 1,
|
||
"fn": 1,
|
||
"tp": 3,
|
||
"accuracy": 0.75,
|
||
"false_reply_rate": 0.25,
|
||
"false_hold_rate": 0.25,
|
||
"recall": 0.75,
|
||
"precision": 0.75,
|
||
"f1": 0.75
|
||
},
|
||
"872a1e67-873a-4c49-a598-b38a670afcad": {
|
||
"n": 9,
|
||
"tn": 3,
|
||
"fp": 1,
|
||
"fn": 1,
|
||
"tp": 4,
|
||
"accuracy": 0.7777777777777778,
|
||
"false_reply_rate": 0.25,
|
||
"false_hold_rate": 0.2,
|
||
"recall": 0.8,
|
||
"precision": 0.8,
|
||
"f1": 0.8
|
||
},
|
||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": {
|
||
"n": 5,
|
||
"tn": 1,
|
||
"fp": 2,
|
||
"fn": 0,
|
||
"tp": 2,
|
||
"accuracy": 0.6,
|
||
"false_reply_rate": 0.6666666666666666,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 0.5,
|
||
"f1": 0.6666666666666666
|
||
},
|
||
"cb3238dc-3946-42a5-aad8-9b9a54226363": {
|
||
"n": 9,
|
||
"tn": 5,
|
||
"fp": 1,
|
||
"fn": 0,
|
||
"tp": 3,
|
||
"accuracy": 0.8888888888888888,
|
||
"false_reply_rate": 0.16666666666666666,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 0.75,
|
||
"f1": 0.8571428571428571
|
||
},
|
||
"e304f740-85d4-427e-9fd6-c5474e9cd737": {
|
||
"n": 7,
|
||
"tn": 3,
|
||
"fp": 0,
|
||
"fn": 1,
|
||
"tp": 3,
|
||
"accuracy": 0.8571428571428571,
|
||
"false_reply_rate": 0.0,
|
||
"false_hold_rate": 0.25,
|
||
"recall": 0.75,
|
||
"precision": 1.0,
|
||
"f1": 0.8571428571428571
|
||
}
|
||
},
|
||
"reason": {
|
||
"completed_answer": {
|
||
"n": 18,
|
||
"tn": 0,
|
||
"fp": 0,
|
||
"fn": 4,
|
||
"tp": 14,
|
||
"accuracy": 0.7777777777777778,
|
||
"false_reply_rate": null,
|
||
"false_hold_rate": 0.2222222222222222,
|
||
"recall": 0.7777777777777778,
|
||
"precision": 1.0,
|
||
"f1": 0.875
|
||
},
|
||
"completed_turn": {
|
||
"n": 4,
|
||
"tn": 0,
|
||
"fp": 0,
|
||
"fn": 0,
|
||
"tp": 4,
|
||
"accuracy": 1.0,
|
||
"false_reply_rate": null,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 1.0,
|
||
"f1": 1.0
|
||
},
|
||
"mid_turn_pause": {
|
||
"n": 17,
|
||
"tn": 12,
|
||
"fp": 5,
|
||
"fn": 0,
|
||
"tp": 0,
|
||
"accuracy": 0.7058823529411765,
|
||
"false_reply_rate": 0.29411764705882354,
|
||
"false_hold_rate": null,
|
||
"recall": null,
|
||
"precision": 0.0,
|
||
"f1": 0.0
|
||
},
|
||
"unfinished_expression": {
|
||
"n": 5,
|
||
"tn": 3,
|
||
"fp": 2,
|
||
"fn": 0,
|
||
"tp": 0,
|
||
"accuracy": 0.6,
|
||
"false_reply_rate": 0.4,
|
||
"false_hold_rate": null,
|
||
"recall": null,
|
||
"precision": 0.0,
|
||
"f1": 0.0
|
||
}
|
||
},
|
||
"label_status": {
|
||
"draft": {
|
||
"n": 29,
|
||
"tn": 7,
|
||
"fp": 4,
|
||
"fn": 4,
|
||
"tp": 14,
|
||
"accuracy": 0.7241379310344828,
|
||
"false_reply_rate": 0.36363636363636365,
|
||
"false_hold_rate": 0.2222222222222222,
|
||
"recall": 0.7777777777777778,
|
||
"precision": 0.7777777777777778,
|
||
"f1": 0.7777777777777778
|
||
},
|
||
"unconfirmed_manual": {
|
||
"n": 15,
|
||
"tn": 8,
|
||
"fp": 3,
|
||
"fn": 0,
|
||
"tp": 4,
|
||
"accuracy": 0.8,
|
||
"false_reply_rate": 0.2727272727272727,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 0.5714285714285714,
|
||
"f1": 0.7272727272727273
|
||
}
|
||
},
|
||
"asr_agent_active_at_point": {
|
||
"0": {
|
||
"n": 43,
|
||
"tn": 15,
|
||
"fp": 6,
|
||
"fn": 4,
|
||
"tp": 18,
|
||
"accuracy": 0.7674418604651163,
|
||
"false_reply_rate": 0.2857142857142857,
|
||
"false_hold_rate": 0.18181818181818182,
|
||
"recall": 0.8181818181818182,
|
||
"precision": 0.75,
|
||
"f1": 0.782608695652174
|
||
},
|
||
"1": {
|
||
"n": 1,
|
||
"tn": 0,
|
||
"fp": 1,
|
||
"fn": 0,
|
||
"tp": 0,
|
||
"accuracy": 0.0,
|
||
"false_reply_rate": 1.0,
|
||
"false_hold_rate": null,
|
||
"recall": null,
|
||
"precision": 0.0,
|
||
"f1": 0.0
|
||
}
|
||
}
|
||
},
|
||
"smart_turn_comparison": {
|
||
"0.5": {
|
||
"n": 44,
|
||
"tn": 4,
|
||
"fp": 18,
|
||
"fn": 0,
|
||
"tp": 22,
|
||
"accuracy": 0.5909090909090909,
|
||
"false_reply_rate": 0.8181818181818182,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 0.55,
|
||
"f1": 0.7096774193548387
|
||
},
|
||
"0.8": {
|
||
"n": 44,
|
||
"tn": 6,
|
||
"fp": 16,
|
||
"fn": 0,
|
||
"tp": 22,
|
||
"accuracy": 0.6363636363636364,
|
||
"false_reply_rate": 0.7272727272727273,
|
||
"false_hold_rate": 0.0,
|
||
"recall": 1.0,
|
||
"precision": 0.5789473684210527,
|
||
"f1": 0.7333333333333333
|
||
}
|
||
},
|
||
"changed_vs_smart_turn": {
|
||
"fixed": 11,
|
||
"regressed": 4
|
||
},
|
||
"config": {
|
||
"maai_version": "0.2.22",
|
||
"torch_version": "2.14.1",
|
||
"device": "cpu",
|
||
"threads": 4,
|
||
"frame_rate": 10,
|
||
"context_len_sec": 20,
|
||
"use_kv_cache": true,
|
||
"weight_variant": "vap_mc",
|
||
"weight_sha256": {
|
||
"vap_mc_state_dict_ch_kyoto_10hz_20000msec.pt": "65b8a51aea6a257b5b9139107f0f0cf22f94523cd4b6a2e7a356b6c43a4596a8",
|
||
"60k_epoch4-d0f474de.pt": "d0f474de968c7243c20f43dee1626681618f2f150d3e638e82b79e22563b17f1"
|
||
},
|
||
"label_sha256": "a809b05bf778532b1b9eb6f64596d7f0a28209869f4fd258e45ede5ce0834db9",
|
||
"history_sha256": "4c21e20b7323030d590276854f07b454ccbad627e1a7842101a22a99377ffbaa",
|
||
"audio_sha256": {
|
||
"2c83253a-7977-4e3c-a387-0aa1ea06fbe1": "2b8a67dc76b8975d287df42193647cf040229377537eadaaf3adc13358201fe1",
|
||
"40ae686c-8196-4d7b-b7ad-3e12a427ebb6": "130be37286d48f7badca711a068fab19a2686bd83248a71c73c11ce77650d39d",
|
||
"872a1e67-873a-4c49-a598-b38a670afcad": "54c5672ea491cc3daedd75e80f97e82cc061ccbf14e521bc22ca55f15fa062a5",
|
||
"a266b9ef-5e47-4434-a04c-ce4eacd52c19": "9a7ebe093e66aaaa08ac462f975b7d428df79a38ed5b418265b82c77d8b91f46",
|
||
"cb3238dc-3946-42a5-aad8-9b9a54226363": "7a74dbf3dd742d3d14317d069c28b703dd1398d5eb3dc1dba3d7a7c568a18d60",
|
||
"e304f740-85d4-427e-9fd6-c5474e9cd737": "44672603d0cf9feac41275e99d59baa4cafbdcdbb172da23de52aa0a2a4961d2"
|
||
},
|
||
"script_sha256": "ca4d5825bd3d86ddc5f41c5fbf1267849e87cabaaa8da9eabd282cd6bbaac2fe"
|
||
},
|
||
"elapsed_seconds": 45.435372250154614,
|
||
"method": "At each exact decision_ms a fresh MaAI/CPC state replays a preceding <=20s stereo window in 100ms frames. Crop before resampling. No future input, no labels as model input, no EOT or VAD gates, no forced timeout replies. Classification is a stated proxy rule, not an official calibrated can_reply output.",
|
||
"limitations": [
|
||
"29 xlsx_draft and 15 rows without explicit verification metadata; exploratory selected-point scores.",
|
||
"The proxy p_now[robot] forecasts relative speaker activity over 0–0.6s; p_future forecasts 0.6–2s. Neither directly predicts task completion.",
|
||
"Independent 20s window restart differs from continuous whole-call state, particularly CPC recurrent state and deeply contextualized KV history.",
|
||
"MaAI uses both recorded channels and up to20s; Smart Turn baseline uses user channel and up to8s. Comparison evaluates these input configurations, not identical-input model capability.",
|
||
"Robot speech already present in recording can influence activity forecasts; agent-active checkpoints are separately marked.",
|
||
"ASR alignment issues previously found at cb3238dc 13540 and e304f740 11531 remain; labels unchanged."
|
||
]
|
||
}
|