wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clarify EasyCom boundary prelabel taxonomy

Browse Source
This commit is contained in:
Xin Wang
2026-09-15 11:41:13 +08:00
parent e2c305facc
commit bb8963b80d
16 changed files with 782 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/easycom_event_ai_prelabel_stats_v15/summary.json
+191
View File
@@ -0,0 +1,191 @@
{
"artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH",
"warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.",
"inputs": [
"reports/easycom_entry_int_session1_v14/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session2_v14/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session3_v14/easycom_eot_int_deepseek_official.jsonl",
"reports/easycom_entry_int_session4_v14/easycom_eot_int_deepseek_official.jsonl"
],
"metadata": {
"protocol_versions": [
"easycom-wearer-entry-int-candidates-v8"
],
"schema_versions": [
"6"
],
"prompt_versions": [
"easycom-wearer-boundary-semantic-prelabel-v15"
],
"models": [
"deepseek-flash"
],
"passes": [
"2"
],
"human_statuses": {
"UNREVIEWED": 2128
}
},
"cohorts": [
{
"cohort": "Session_1",
"sessions": [
"Session_1"
],
"num_candidates": 496,
"num_entry_family": 285,
"num_int_family": 211,
"num_positive": 116,
"num_hard_negative": 182,
"num_exclude": 198,
"num_scoring_eligible_prelabels": 298,
"scoring_eligible_prelabel_percent": 60.08,
"mean_ai_confidence": 0.7012,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 496,
"num_two_pass_semantic_agreement": 458,
"two_pass_semantic_agreement_percent": 92.34,
"num_two_pass_full_decision_agreement": 453,
"two_pass_full_decision_agreement_percent": 91.33,
"num_human_reviewed": 0
},
{
"cohort": "Session_2",
"sessions": [
"Session_2"
],
"num_candidates": 490,
"num_entry_family": 334,
"num_int_family": 156,
"num_positive": 71,
"num_hard_negative": 252,
"num_exclude": 167,
"num_scoring_eligible_prelabels": 323,
"scoring_eligible_prelabel_percent": 65.92,
"mean_ai_confidence": 0.7143,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 490,
"num_two_pass_semantic_agreement": 454,
"two_pass_semantic_agreement_percent": 92.65,
"num_two_pass_full_decision_agreement": 452,
"two_pass_full_decision_agreement_percent": 92.24,
"num_human_reviewed": 0
},
{
"cohort": "Session_3",
"sessions": [
"Session_3"
],
"num_candidates": 545,
"num_entry_family": 308,
"num_int_family": 237,
"num_positive": 117,
"num_hard_negative": 184,
"num_exclude": 244,
"num_scoring_eligible_prelabels": 301,
"scoring_eligible_prelabel_percent": 55.23,
"mean_ai_confidence": 0.7256,
"median_ai_confidence": 0.75,
"num_rows_with_two_passes": 545,
"num_two_pass_semantic_agreement": 489,
"two_pass_semantic_agreement_percent": 89.72,
"num_two_pass_full_decision_agreement": 486,
"two_pass_full_decision_agreement_percent": 89.17,
"num_human_reviewed": 0
},
{
"cohort": "Session_4",
"sessions": [
"Session_4"
],
"num_candidates": 597,
"num_entry_family": 471,
"num_int_family": 126,
"num_positive": 84,
"num_hard_negative": 381,
"num_exclude": 132,
"num_scoring_eligible_prelabels": 465,
"scoring_eligible_prelabel_percent": 77.89,
"mean_ai_confidence": 0.7348,
"median_ai_confidence": 0.75,
"num_rows_with_two_passes": 597,
"num_two_pass_semantic_agreement": 575,
"two_pass_semantic_agreement_percent": 96.31,
"num_two_pass_full_decision_agreement": 571,
"two_pass_full_decision_agreement_percent": 95.64,
"num_human_reviewed": 0
},
{
"cohort": "Test pooled (Sessions 1-3)",
"sessions": [
"Session_1",
"Session_2",
"Session_3"
],
"num_candidates": 1531,
"num_entry_family": 927,
"num_int_family": 604,
"num_positive": 304,
"num_hard_negative": 618,
"num_exclude": 609,
"num_scoring_eligible_prelabels": 922,
"scoring_eligible_prelabel_percent": 60.22,
"mean_ai_confidence": 0.7141,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 1531,
"num_two_pass_semantic_agreement": 1401,
"two_pass_semantic_agreement_percent": 91.51,
"num_two_pass_full_decision_agreement": 1391,
"two_pass_full_decision_agreement_percent": 90.86,
"num_human_reviewed": 0
},
{
"cohort": "Session 4 audit (train split)",
"sessions": [
"Session_4"
],
"num_candidates": 597,
"num_entry_family": 471,
"num_int_family": 126,
"num_positive": 84,
"num_hard_negative": 381,
"num_exclude": 132,
"num_scoring_eligible_prelabels": 465,
"scoring_eligible_prelabel_percent": 77.89,
"mean_ai_confidence": 0.7348,
"median_ai_confidence": 0.75,
"num_rows_with_two_passes": 597,
"num_two_pass_semantic_agreement": 575,
"two_pass_semantic_agreement_percent": 96.31,
"num_two_pass_full_decision_agreement": 571,
"two_pass_full_decision_agreement_percent": 95.64,
"num_human_reviewed": 0
},
{
"cohort": "All supplied (descriptive only)",
"sessions": [
"Session_1",
"Session_2",
"Session_3",
"Session_4"
],
"num_candidates": 2128,
"num_entry_family": 1398,
"num_int_family": 730,
"num_positive": 388,
"num_hard_negative": 999,
"num_exclude": 741,
"num_scoring_eligible_prelabels": 1387,
"scoring_eligible_prelabel_percent": 65.18,
"mean_ai_confidence": 0.7199,
"median_ai_confidence": 0.7,
"num_rows_with_two_passes": 2128,
"num_two_pass_semantic_agreement": 1976,
"two_pass_semantic_agreement_percent": 92.86,
"num_two_pass_full_decision_agreement": 1962,
"two_pass_full_decision_agreement_percent": 92.2,
"num_human_reviewed": 0
}
]
}