Record EgoSpeak three-level evaluation

2026-09-16 03:31:10 +08:00
parent e4b429545c
commit 4b4624f6b8
8 changed files with 186 additions and 2 deletions
@@ -0,0 +1,76 @@
+# EasyCom EgoSpeak three-level evaluation
+
+This record freezes the first Sessions 1--3 evaluation of the released
+EgoSpeak MiniROADA audio-visual checkpoint under three deliberately separate
+questions. The values below were produced on the remote experiment host; the
+aggregate console output was copied into this record on 2026-09-16. Generated
+prediction and per-event artifacts remain on that host and are not committed.
+
+## Protocol and split
+
+- Test split: EasyCom Sessions 1--3.
+- Model: released EgoSpeak MiniROADA audio-visual checkpoint.
+- Dense forecasting: framewise AP for background, wearer, and aggregate
+  others at ten horizons from 0.2 to 2.0 seconds.
+- Raw boundary anticipation: threshold-free AP for wearer `OFF->ON` and
+  `ON->OFF` changes. These are timing-derived diagnostics, not semantic EOT or
+  INT claims.
+- Opportunity-conditioned behavior: one causally censored activity-derived
+  score per reviewed ENTRY or INT/YIELD opportunity. Labels are provisional
+  two-pass LLM-assisted reviews and have not been human reviewed.
+- No test-set threshold is fitted for any AP reported here. Session 4 is needed
+  only for threshold-dependent operating points, which are intentionally not
+  summarized as headline findings in this record.
+
+## Findings
+
+Dense forecasting reaches **74.124% mAP at 0.2 s** and **61.155% mean mAP**
+over ten horizons. The wearer class alone reaches 63.750% AP at 0.2 s and
+53.493% averaged over horizons. Restricting evaluation to causally valid rows
+changes mean mAP by only +0.038 percentage points (61.193% versus 61.155%).
+
+Raw wearer onset and offset anticipation obtain **24.491%** and **38.024%**
+mean AP, respectively. These lower values do not contradict the dense result:
+dense AP rewards correct future state over sustained regions, whereas a raw
+boundary event is counted once and has many harder non-boundary comparisons.
+
+On the provisional opportunity labels, ENTRY AP is **52.487%** for 163
+positives and 335 hard negatives, versus 32.731% positive prevalence
+(+19.756 points; 1.60x prevalence). INT/YIELD AP is **20.135%** for 28 positives
+and 109 hard negatives, essentially equal to its 20.438% prevalence. This is
+consistent with the output contract: wearer probability can rank many entry
+opportunities, but `1-p(wearer)` does not encode whether an interruption should
+cause a yield. The ENTRY and INT loaders additionally exclude 81 and 5
+invalid/post-outcome cases; these exclusions require an audit before any final
+paper claim.
+
+## Relation to EgoSpeak Figure 7
+
+Figure 7 of EgoSpeak plots a continuous target/wearer-speaking confidence over
+ground-truth speaking regions. It therefore corresponds most directly to the
+dense **wearer AP**, not to raw onset/offset AP or semantic ENTRY/YIELD AP. A
+visually convincing dense curve can coexist with weak boundary or semantic
+selectivity. The figure caption does not identify one forecast horizon, and it
+uses the paper's Transformer variants rather than proving the aggregate result
+of this MiniROADA run.
+
+For our eventual qualitative figure, the continuous wearer curve can be kept,
+but it should also show the causal opportunity anchor, event boundary,
+competing-risk censor, and the single score assigned to each positive or hard
+negative.
+
+## Interpretation limits
+
+- The three metric layers answer different questions and their similar numeric
+  values must not be equated.
+- Aggregate `others` activity cannot identify a particular speaker, addressee,
+  semantic EOT, or interruption intent.
+- The opportunity result is provisional until label/exclusion audit and human
+  review; it is useful for baseline diagnosis, not a final confirmatory claim.
+- This is one released checkpoint, not a multi-seed uncertainty estimate.
+
+Machine-readable curves and counts are in [`summary.json`](summary.json) and
+the compact comparison is in [`metrics.csv`](metrics.csv). The opportunity
+protocol and reproduction command are documented in
+[`docs/easycom_opportunity_event_evaluation.md`](../../docs/easycom_opportunity_event_evaluation.md).
+
@@ -0,0 +1,10 @@
+layer,metric,horizon_seconds,value_percent,num_positive,num_hard_negative,positive_prevalence_percent
+dense,dense_mAP,0.2,74.124,,,
+dense,dense_mean_mAP,mean,61.155,,,
+dense,wearer_AP,0.2,63.750,,,
+dense,wearer_mean_AP,mean,53.493,,,
+raw_boundary,wearer_onset_mean_AP,mean,24.491,,,
+raw_boundary,wearer_offset_mean_AP,mean,38.024,,,
+opportunity,ENTRY_AP,opportunity,52.4867,163,335,32.7309
+opportunity,INT_YIELD_AP,opportunity,20.1349,28,109,20.4380
+
@@ -0,0 +1,60 @@
+{
+  "recorded_at": "2026-09-16",
+  "source": "user-provided remote experiment output",
+  "dataset": "EasyCom",
+  "sessions": ["Session_1", "Session_2", "Session_3"],
+  "model": "EgoSpeak MiniROADA audio-visual released checkpoint",
+  "scale": "percent",
+  "dense_forecasting": {
+    "horizons_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0],
+    "full_timeline": {
+      "map": [74.124, 68.291, 64.267, 61.091, 59.734, 58.774, 57.471, 57.064, 55.933, 54.802],
+      "background_ap": [67.957, 58.434, 52.256, 47.545, 46.106, 45.787, 43.552, 43.491, 42.212, 40.345],
+      "wearer_ap": [63.750, 59.595, 56.614, 54.163, 52.903, 51.471, 50.745, 49.721, 48.587, 47.376],
+      "others_ap": [90.666, 86.845, 83.930, 81.565, 80.192, 79.062, 78.116, 77.982, 76.998, 76.684],
+      "mean_map": 61.155,
+      "mean_wearer_ap": 53.493
+    },
+    "causal_valid_only": {
+      "map": [74.204, 68.349, 64.313, 61.128, 59.768, 58.805, 57.498, 57.090, 55.954, 54.821],
+      "background_ap": [68.189, 58.596, 52.373, 47.629, 46.172, 45.840, 43.587, 43.518, 42.227, 40.349],
+      "wearer_ap": [63.750, 59.595, 56.619, 54.172, 52.916, 51.488, 50.765, 49.739, 48.605, 47.393],
+      "others_ap": [90.673, 86.857, 83.947, 81.585, 80.216, 79.088, 78.143, 78.012, 77.031, 76.720],
+      "mean_map": 61.193,
+      "mean_wearer_ap": 53.504
+    }
+  },
+  "raw_boundary_anticipation": {
+    "horizons_seconds": [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 1.4, 1.6, 1.8, 2.0],
+    "wearer_onset_ap": [5.139, 9.878, 15.389, 21.093, 27.225, 30.997, 33.563, 34.046, 34.030, 33.550],
+    "wearer_offset_ap": [9.696, 18.846, 27.730, 34.971, 40.646, 45.112, 47.880, 50.444, 52.051, 52.862],
+    "mean_wearer_onset_ap": 24.491,
+    "mean_wearer_offset_ap": 38.024,
+    "timing_derived_eot_hold_mean_ap": 9.872,
+    "timing_derived_yield_mean_ap": 47.688
+  },
+  "opportunity_conditioned_behavior": {
+    "labels": "provisional two-pass LLM-assisted; not human reviewed",
+    "entry": {
+      "positive": 163,
+      "hard_negative": 335,
+      "invalid_or_post_outcome": 81,
+      "average_precision": 52.4867,
+      "positive_prevalence": 32.7309,
+      "ap_lift_points": 19.7558,
+      "ap_over_prevalence": 1.6036
+    },
+    "int_yield": {
+      "positive": 28,
+      "hard_negative": 109,
+      "invalid_or_post_outcome": 5,
+      "average_precision": 20.1349,
+      "positive_prevalence": 20.4380,
+      "ap_lift_points": -0.3031,
+      "ap_over_prevalence": 0.9852
+    }
+  },
+  "threshold_free": true,
+  "session4_required_for_reported_metrics": false
+}
+