192 lines
5.9 KiB
JSON
192 lines
5.9 KiB
JSON
{
|
|
"artifact_type": "AI_PRELABEL_DESCRIPTIVE_STATISTICS_NOT_GROUND_TRUTH",
|
|
"warning": "All supplied rows require human adjudication. Do not place these counts in the manuscript table labeled after human adjudication.",
|
|
"inputs": [
|
|
"reports/easycom_entry_int_session1_v14/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session2_v14/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session3_v14/easycom_eot_int_deepseek_official.jsonl",
|
|
"reports/easycom_entry_int_session4_v14/easycom_eot_int_deepseek_official.jsonl"
|
|
],
|
|
"metadata": {
|
|
"protocol_versions": [
|
|
"easycom-wearer-entry-int-candidates-v8"
|
|
],
|
|
"schema_versions": [
|
|
"6"
|
|
],
|
|
"prompt_versions": [
|
|
"easycom-wearer-boundary-semantic-prelabel-v15"
|
|
],
|
|
"models": [
|
|
"deepseek-flash"
|
|
],
|
|
"passes": [
|
|
"2"
|
|
],
|
|
"human_statuses": {
|
|
"UNREVIEWED": 2128
|
|
}
|
|
},
|
|
"cohorts": [
|
|
{
|
|
"cohort": "Session_1",
|
|
"sessions": [
|
|
"Session_1"
|
|
],
|
|
"num_candidates": 496,
|
|
"num_entry_family": 285,
|
|
"num_int_family": 211,
|
|
"num_positive": 116,
|
|
"num_hard_negative": 182,
|
|
"num_exclude": 198,
|
|
"num_scoring_eligible_prelabels": 298,
|
|
"scoring_eligible_prelabel_percent": 60.08,
|
|
"mean_ai_confidence": 0.7012,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 496,
|
|
"num_two_pass_semantic_agreement": 458,
|
|
"two_pass_semantic_agreement_percent": 92.34,
|
|
"num_two_pass_full_decision_agreement": 453,
|
|
"two_pass_full_decision_agreement_percent": 91.33,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_2",
|
|
"sessions": [
|
|
"Session_2"
|
|
],
|
|
"num_candidates": 490,
|
|
"num_entry_family": 334,
|
|
"num_int_family": 156,
|
|
"num_positive": 71,
|
|
"num_hard_negative": 252,
|
|
"num_exclude": 167,
|
|
"num_scoring_eligible_prelabels": 323,
|
|
"scoring_eligible_prelabel_percent": 65.92,
|
|
"mean_ai_confidence": 0.7143,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 490,
|
|
"num_two_pass_semantic_agreement": 454,
|
|
"two_pass_semantic_agreement_percent": 92.65,
|
|
"num_two_pass_full_decision_agreement": 452,
|
|
"two_pass_full_decision_agreement_percent": 92.24,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_3",
|
|
"sessions": [
|
|
"Session_3"
|
|
],
|
|
"num_candidates": 545,
|
|
"num_entry_family": 308,
|
|
"num_int_family": 237,
|
|
"num_positive": 117,
|
|
"num_hard_negative": 184,
|
|
"num_exclude": 244,
|
|
"num_scoring_eligible_prelabels": 301,
|
|
"scoring_eligible_prelabel_percent": 55.23,
|
|
"mean_ai_confidence": 0.7256,
|
|
"median_ai_confidence": 0.75,
|
|
"num_rows_with_two_passes": 545,
|
|
"num_two_pass_semantic_agreement": 489,
|
|
"two_pass_semantic_agreement_percent": 89.72,
|
|
"num_two_pass_full_decision_agreement": 486,
|
|
"two_pass_full_decision_agreement_percent": 89.17,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session_4",
|
|
"sessions": [
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 597,
|
|
"num_entry_family": 471,
|
|
"num_int_family": 126,
|
|
"num_positive": 84,
|
|
"num_hard_negative": 381,
|
|
"num_exclude": 132,
|
|
"num_scoring_eligible_prelabels": 465,
|
|
"scoring_eligible_prelabel_percent": 77.89,
|
|
"mean_ai_confidence": 0.7348,
|
|
"median_ai_confidence": 0.75,
|
|
"num_rows_with_two_passes": 597,
|
|
"num_two_pass_semantic_agreement": 575,
|
|
"two_pass_semantic_agreement_percent": 96.31,
|
|
"num_two_pass_full_decision_agreement": 571,
|
|
"two_pass_full_decision_agreement_percent": 95.64,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Test pooled (Sessions 1-3)",
|
|
"sessions": [
|
|
"Session_1",
|
|
"Session_2",
|
|
"Session_3"
|
|
],
|
|
"num_candidates": 1531,
|
|
"num_entry_family": 927,
|
|
"num_int_family": 604,
|
|
"num_positive": 304,
|
|
"num_hard_negative": 618,
|
|
"num_exclude": 609,
|
|
"num_scoring_eligible_prelabels": 922,
|
|
"scoring_eligible_prelabel_percent": 60.22,
|
|
"mean_ai_confidence": 0.7141,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 1531,
|
|
"num_two_pass_semantic_agreement": 1401,
|
|
"two_pass_semantic_agreement_percent": 91.51,
|
|
"num_two_pass_full_decision_agreement": 1391,
|
|
"two_pass_full_decision_agreement_percent": 90.86,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "Session 4 audit (train split)",
|
|
"sessions": [
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 597,
|
|
"num_entry_family": 471,
|
|
"num_int_family": 126,
|
|
"num_positive": 84,
|
|
"num_hard_negative": 381,
|
|
"num_exclude": 132,
|
|
"num_scoring_eligible_prelabels": 465,
|
|
"scoring_eligible_prelabel_percent": 77.89,
|
|
"mean_ai_confidence": 0.7348,
|
|
"median_ai_confidence": 0.75,
|
|
"num_rows_with_two_passes": 597,
|
|
"num_two_pass_semantic_agreement": 575,
|
|
"two_pass_semantic_agreement_percent": 96.31,
|
|
"num_two_pass_full_decision_agreement": 571,
|
|
"two_pass_full_decision_agreement_percent": 95.64,
|
|
"num_human_reviewed": 0
|
|
},
|
|
{
|
|
"cohort": "All supplied (descriptive only)",
|
|
"sessions": [
|
|
"Session_1",
|
|
"Session_2",
|
|
"Session_3",
|
|
"Session_4"
|
|
],
|
|
"num_candidates": 2128,
|
|
"num_entry_family": 1398,
|
|
"num_int_family": 730,
|
|
"num_positive": 388,
|
|
"num_hard_negative": 999,
|
|
"num_exclude": 741,
|
|
"num_scoring_eligible_prelabels": 1387,
|
|
"scoring_eligible_prelabel_percent": 65.18,
|
|
"mean_ai_confidence": 0.7199,
|
|
"median_ai_confidence": 0.7,
|
|
"num_rows_with_two_passes": 2128,
|
|
"num_two_pass_semantic_agreement": 1976,
|
|
"two_pass_semantic_agreement_percent": 92.86,
|
|
"num_two_pass_full_decision_agreement": 1962,
|
|
"two_pass_full_decision_agreement_percent": 92.2,
|
|
"num_human_reviewed": 0
|
|
}
|
|
]
|
|
}
|