wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add EasyCom v16 AI prelabel results

Browse Source
This commit is contained in:
Xin Wang
2026-09-15 12:59:03 +08:00
parent bb8963b80d
commit c710b9e023
20 changed files with 2821 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/easycom_event_ai_prelabel_stats_v16/summary.json
+191
View File
@@ -0,0 +1,191 @@
{
"artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH",
"warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.",
"inputs": [
"reports/easycom_entry_int_session1_v16/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session2_v16/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session3_v16/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session4_v16/easycom_eot_int_deepseek_official.jsonl"
],
"metadata": {
"protocol_versions": [
"easycom-wearer-entry-int-candidates-v8"
],
"schema_versions": [
"6"
],
"prompt_versions": [
"easycom-wearer-boundary-semantic-prelabel-v16"
],
"models": [
"deepseek-flash"
],
"passes": [
"2"
],
"human_statuses": {
"UNREVIEWED": 2128
}
},
"cohorts": [
{
"cohort": "Session_1",
"sessions": [
"Session_1"
],
"num_candidates": 496,
"num_entry_family": 285,
"num_int_family": 211,
"num_positive": 113,
"num_hard_negative": 192,
"num_exclude": 191,
"num_scoring_eligible_prelabels": 305,
"scoring_eligible_prelabel_percent": 61.49,
"mean_ai_confidence": 0.6863,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 496,
"num_two_pass_semantic_agreement": 458,
"two_pass_semantic_agreement_percent": 92.34,
"num_two_pass_full_decision_agreement": 456,
"two_pass_full_decision_agreement_percent": 91.94,
"num_human_reviewed": 0
},
{
"cohort": "Session_2",
"sessions": [
"Session_2"
],
"num_candidates": 490,
"num_entry_family": 334,
"num_int_family": 156,
"num_positive": 64,
"num_hard_negative": 260,
"num_exclude": 166,
"num_scoring_eligible_prelabels": 324,
"scoring_eligible_prelabel_percent": 66.12,
"mean_ai_confidence": 0.6966,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 490,
"num_two_pass_semantic_agreement": 467,
"two_pass_semantic_agreement_percent": 95.31,
"num_two_pass_full_decision_agreement": 464,
"two_pass_full_decision_agreement_percent": 94.69,
"num_human_reviewed": 0
},
{
"cohort": "Session_3",
"sessions": [
"Session_3"
],
"num_candidates": 545,
"num_entry_family": 308,
"num_int_family": 237,
"num_positive": 112,
"num_hard_negative": 200,
"num_exclude": 233,
"num_scoring_eligible_prelabels": 312,
"scoring_eligible_prelabel_percent": 57.25,
"mean_ai_confidence": 0.7134,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 545,
"num_two_pass_semantic_agreement": 507,
"two_pass_semantic_agreement_percent": 93.03,
"num_two_pass_full_decision_agreement": 504,
"two_pass_full_decision_agreement_percent": 92.48,
"num_human_reviewed": 0
},
{
"cohort": "Session_4",
"sessions": [
"Session_4"
],
"num_candidates": 597,
"num_entry_family": 471,
"num_int_family": 126,
"num_positive": 78,
"num_hard_negative": 365,
"num_exclude": 154,
"num_scoring_eligible_prelabels": 443,
"scoring_eligible_prelabel_percent": 74.2,
"mean_ai_confidence": 0.6997,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 597,
"num_two_pass_semantic_agreement": 555,
"two_pass_semantic_agreement_percent": 92.96,
"num_two_pass_full_decision_agreement": 550,
"two_pass_full_decision_agreement_percent": 92.13,
"num_human_reviewed": 0
},
{
"cohort": "Test pooled (Sessions 1-3)",
"sessions": [
"Session_1",
"Session_2",
"Session_3"
],
"num_candidates": 1531,
"num_entry_family": 927,
"num_int_family": 604,
"num_positive": 289,
"num_hard_negative": 652,
"num_exclude": 590,
"num_scoring_eligible_prelabels": 941,
"scoring_eligible_prelabel_percent": 61.46,
"mean_ai_confidence": 0.6992,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 1531,
"num_two_pass_semantic_agreement": 1432,
"two_pass_semantic_agreement_percent": 93.53,
"num_two_pass_full_decision_agreement": 1424,
"two_pass_full_decision_agreement_percent": 93.01,
"num_human_reviewed": 0
},
{
"cohort": "Session 4 audit (train split)",
"sessions": [
"Session_4"
],
"num_candidates": 597,
"num_entry_family": 471,
"num_int_family": 126,
"num_positive": 78,
"num_hard_negative": 365,
"num_exclude": 154,
"num_scoring_eligible_prelabels": 443,
"scoring_eligible_prelabel_percent": 74.2,
"mean_ai_confidence": 0.6997,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 597,
"num_two_pass_semantic_agreement": 555,
"two_pass_semantic_agreement_percent": 92.96,
"num_two_pass_full_decision_agreement": 550,
"two_pass_full_decision_agreement_percent": 92.13,
"num_human_reviewed": 0
},
{
"cohort": "All supplied (descriptive only)",
"sessions": [
"Session_1",
"Session_2",
"Session_3",
"Session_4"
],
"num_candidates": 2128,
"num_entry_family": 1398,
"num_int_family": 730,
"num_positive": 367,
"num_hard_negative": 1017,
"num_exclude": 744,
"num_scoring_eligible_prelabels": 1384,
"scoring_eligible_prelabel_percent": 65.04,
"mean_ai_confidence": 0.6994,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 2128,
"num_two_pass_semantic_agreement": 1987,
"two_pass_semantic_agreement_percent": 93.37,
"num_two_pass_full_decision_agreement": 1974,
"two_pass_full_decision_agreement_percent": 92.76,
"num_human_reviewed": 0
}
]
}