From d4c900ab130a1d6f2f0daa1182998ddedba60634 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 16 Sep 2026 04:31:18 +0800 Subject: [PATCH] Fix opportunity competing-risk evaluation --- PROJECT_CONTEXT.md | 29 ++-- docs/easycom_opportunity_event_evaluation.md | 37 ++-- ...peaker_floor_experiment_plan_2026-08-30.md | 13 +- evaluate_egospeak_opportunity_events.py | 161 +++++++++++++++--- manuscript/main/main.tex | 13 +- .../README.md | 24 ++- .../summary.json | 3 +- .../README.md | 15 +- .../summary.json | 3 +- test_evaluate_egospeak_opportunity_events.py | 100 ++++++++++- 10 files changed, 310 insertions(+), 88 deletions(-) diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index ed3e0d1..3fa9863 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -54,19 +54,23 @@ result. - An opportunity-conditioned dense-output evaluator aligns ENTRY positives and core negatives at non-wearer releases, aligns INT positives and hold-floor negatives at competing non-wearer onsets, and applies strict competing-risk - censoring. It uses only the original wearer probabilities, excludes wearer - backchannels from headline ENTRY FPR, and records unanchored/ambiguous cases. + censoring. ENTRY positives whose selected release is followed by a competing + other onset or intervening wearer activity are excluded, and ambiguous + multi-competitor INT cases are not assigned an arbitrary onset. It uses only + the original wearer probabilities, excludes wearer backchannels from + headline ENTRY FPR, and records unanchored/ambiguous cases. ### Measured screening result - The released EgoSpeak MiniROADA A+V checkpoint on Sessions 1--3 obtains `74.124` 0.2-second mAP and `61.155` ten-horizon mean mAP. Raw wearer onset/offset anticipation is harder (`24.491/38.024` mean AP). On provisional - LLM-assisted matched opportunities, ENTRY AP is `52.487` versus `32.731%` - prevalence, while INT/YIELD AP is `20.135`, indistinguishable from its - `20.438%` prevalence. Figure-7-style continuous wearer curves therefore - support dense forecasting, not semantic floor-control claims. Full curves, - counts, exclusions, and caveats are frozen in + LLM-assisted matched opportunities, the original v1 evaluator reported + ENTRY/INT AP of `52.487/20.135`; those opportunity values are superseded + because v1 could cross a competing ENTRY onset and arbitrarily selected among + multiple INT competitors. Figure-7-style continuous wearer curves support + dense forecasting, not semantic floor-control claims. The unaffected dense + curves and historical v1 opportunity record are in [`results/easycom_egospeak_three_level_eval_2026-09-16/`](results/easycom_egospeak_three_level_eval_2026-09-16/README.md). - F1 Frame-set Attention uses fused G+A as one query over the latest half-open causal person set. On the existing three-seed, four-epoch test screen it @@ -74,12 +78,11 @@ result. `64.95099 +/- 0.09451` ten-horizon mean mAP. This is stable, but is not a matched improvement over the historical no-dev pooled result. Its three-seed boundary response/yield AP is `28.423 +/- 0.480` / `48.641 +/- 0.391`; on - provisional reviewed opportunities its ENTRY/INT AP is - `41.561 +/- 0.581` / `22.991 +/- 0.202`. These values are respectively - `-4.461/-1.275` points below the historical pooled-V seed-0 result, so the - current F1 screen does not support the hoped-for semantic-selectivity gain. - A same-split three-seed pooled control and correct/wrong visual controls - remain missing. See the immutable + the superseded v1 opportunity evaluator reported ENTRY/INT AP of + `41.561 +/- 0.581` / `22.991 +/- 0.202`; these values require a v2 rerun and + cannot currently support a semantic-selectivity conclusion. A same-split + three-seed pooled control and correct/wrong visual controls remain missing. + See the historical [`screening record`](results/easycom_omni_floor_screen_2026-09-16/README.md). ### Measured local-fusion screens diff --git a/docs/easycom_opportunity_event_evaluation.md b/docs/easycom_opportunity_event_evaluation.md index 9f8b09b..be5c9fc 100644 --- a/docs/easycom_opportunity_event_evaluation.md +++ b/docs/easycom_opportunity_event_evaluation.md @@ -10,6 +10,12 @@ head. A positive must have a reviewer-selected preceding non-wearer release. Its opportunity starts at that release and resolves at the reviewed wearer entry. +If a new non-wearer onset occurs before or exactly at that entry, the release +has already been resolved by a competing event and the positive is excluded. +The same applies when wearer activity already occurred between the selected +release and the reviewed onset: the selected release is then a stale, +post-outcome anchor. Both cases are counted separately in +`review_diagnostics.skipped`. Core hard negatives start at the same type of non-wearer release and resolve when the same speaker resumes (`ENTRY_OTHER_MIDTURN_PAUSE`) or a different non-wearer takes the next floor (`ENTRY_OTHER_TO_OTHER`). The score is the @@ -27,10 +33,10 @@ the maximum `1 - p_wearer` over 0.2--0.6 seconds, censored strictly at that non-wearer's offset. Reviewed `INT_YIELD` items are positive; other backchannels, failed takeovers, and clear wearer-hold cases are hard negatives. -Current INT reviews do not select one competing onset. The loader therefore -uses the latest supplied other interval that crosses the wearer offset within -the annotation tolerance. It records how many intervals were eligible so -multi-competitor cases can be audited before the protocol is frozen. +Current INT reviews do not select one competing onset. A unique supplied other +interval that crosses the wearer offset within the annotation tolerance is +used. Cases with multiple eligible intervals are excluded and counted rather +than being assigned a deterministic latest-onset anchor. ## Provisional EgoSpeak run @@ -58,21 +64,24 @@ Before manuscript reporting, freeze the 2.0-second ENTRY cap, 0.6-second YIELD cap, positive anchor-selection policy, and multiple-competitor handling using training-side data only. -## Recorded Sessions 1--3 baseline +## Superseded v1 Sessions 1--3 baseline -The first threshold-free run gives ENTRY AP 52.487% (163 positive, 335 hard -negative; 32.731% prevalence) and INT/YIELD AP 20.135% (28 positive, 109 hard -negative; 20.438% prevalence). It excludes 81 ENTRY and 5 INT cases as invalid -or post-outcome. These are provisional LLM-assisted labels, not human-reviewed -confirmatory results. Dense, raw-boundary, and opportunity metrics and their -Figure 7 interpretation are frozen together in the +The first threshold-free run used the superseded v1 loader and reported ENTRY +AP 52.487% (163 positive, 335 hard negative; 32.731% prevalence) and INT/YIELD +AP 20.135% (28 positive, 109 hard negative; 20.438% prevalence). It excluded +81 ENTRY and 5 INT cases as invalid or post-outcome. These opportunity values +must not be reported: v1 allowed a +selected ENTRY release to cross an intervening other onset and selected the +latest onset in ambiguous multi-competitor INT cases. Dense and raw-boundary +metrics are unaffected. Rerun the opportunity evaluation with protocol v2 +before updating the manuscript. The historical output is retained in the [`2026-09-16 result record`](../results/easycom_egospeak_three_level_eval_2026-09-16/README.md). -The expanded Omni screen finds no opportunity-ranking gain from the current +The expanded v1 Omni screen found no opportunity-ranking gain from the current F1 Frame-set Attention model: its three-seed ENTRY/INT AP is 41.561%/22.991%, below the historical pooled-V seed-0 values 46.022%/24.266%. Because F1 and that pooled run use different train/dev model- -selection protocols, this is a stable negative screen rather than a matched -confirmatory comparison. Dense, boundary, per-seed, and provenance details are +selection protocols and used the superseded opportunity loader, only its dense +and raw-boundary findings remain current. Per-seed and provenance details are in the [`Omni floor screening record`](../results/easycom_omni_floor_screen_2026-09-16/README.md). diff --git a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md index 50505a9..c459bb0 100644 --- a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md +++ b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md @@ -168,12 +168,13 @@ forecasting 的定性解释，不能代替 boundary 或 semantic opportunity 评 曲线和限制见 [`three-level baseline record`](../results/easycom_egospeak_three_level_eval_2026-09-16/README.md)。 -当前 F1 current-frame person-set screen 的三种子结果同样没有支持 semantic floor -selectivity：ENTRY/INT opportunity AP 为 `41.561 +/- 0.581` / `22.991 +/- 0.202`， -低于历史 pooled-V seed-0 的 `46.022/24.266`；dense mean mAP 也为 -`64.951 +/- 0.095`，没有超过历史 pooled-V 的 `65.400`。由于两者 train/dev -选择协议不同，这只能判为稳定的负向 screening，正式归因前必须补同协议三种子 pooled -control。详见 [`Omni floor screening record`](../results/easycom_omni_floor_screen_2026-09-16/README.md)。 +当前 F1 current-frame person-set screen 的 dense mean mAP 为 +`64.951 +/- 0.095`，没有超过历史 pooled-V 的 `65.400`。原 v1 evaluator 给出的 +ENTRY/INT opportunity AP（`41.561 +/- 0.581` / `22.991 +/- 0.202`）已失效：v1 +可能跨过 competing ENTRY onset，且会在多个 INT competitor 中任意选择 latest onset。 +必须用 v2 重跑后才能判断 semantic floor selectivity；正式归因还必须补同协议三种子 +pooled control。历史数值与勘误见 +[`Omni floor screening record`](../results/easycom_omni_floor_screen_2026-09-16/README.md)。 ### P3：第一优先方法——匿名逐人时序旁路 diff --git a/evaluate_egospeak_opportunity_events.py b/evaluate_egospeak_opportunity_events.py index c0d8fe6..dbf9f46 100755 --- a/evaluate_egospeak_opportunity_events.py +++ b/evaluate_egospeak_opportunity_events.py @@ -8,14 +8,18 @@ event head. ENTRY opportunities are anchored at a reviewed preceding non-wearer release. Positive opportunities resolve at the wearer entry; core hard negatives -resolve when the same or a different non-wearer speaks next. The score is the -maximum wearer probability over available 0.2--2.0 s forecasts up to that -competing resolution. +resolve when the same or a different non-wearer speaks next. A positive is +excluded if another non-wearer onset competes before (or exactly at) its wearer +entry, or if wearer activity already occurred after the selected release. The +score is the maximum wearer probability over available 0.2--2.0 s forecasts up +to the uncontested resolution. YIELD opportunities are anchored at a non-wearer onset during wearer speech. The score is the maximum future wearer-inactivity probability over 0.2--0.6 s, -censored at that non-wearer's offset. Positive and hard-negative examples -therefore begin at the same kind of observed conversational opportunity. +censored at that non-wearer's offset. Reviews with multiple eligible +competing onsets are excluded until the annotation selects one. Positive and +hard-negative examples therefore begin at the same kind of unambiguous +observed conversational opportunity. Prediction arrays use the repository contract ``[target_time, class, future_horizon]``. All scores are read diagonally from @@ -178,17 +182,83 @@ def _candidate_region( return start, end -def _int_competing_interval( +def _entry_selected_option( + candidate: dict[str, Any], + review: dict[str, Any], + *, + selected_release_seconds: float, + uid: str, +) -> dict[str, Any] | None: + evidence = candidate.get("temporal_evidence") + evidence = evidence if isinstance(evidence, dict) else {} + options = evidence.get("opportunity_options") + if not isinstance(options, list): + return None + selected_uid = review.get("selected_opportunity_uid") + matches: list[dict[str, Any]] = [] + for index, option in enumerate(options): + if not isinstance(option, dict): + continue + option_seconds = _optional_seconds( + option.get("anchor_seconds"), + field=f"opportunity_options[{index}].anchor_seconds", + uid=uid, + ) + if selected_uid is not None and option.get("uid") == selected_uid: + matches.append(option) + elif selected_uid is None and option_seconds is not None and math.isclose( + option_seconds, selected_release_seconds, abs_tol=1e-7 + ): + matches.append(option) + if len(matches) > 1: + raise ValueError(f"{uid}: selected ENTRY opportunity is not unique") + return matches[0] if matches else None + + +def _entry_competing_onsets( + candidate: dict[str, Any], + *, + selected_release_seconds: float, + behavior_anchor_seconds: float, + uid: str, +) -> list[float]: + """Return other onsets that resolve a release before the wearer entry.""" + + evidence = candidate.get("temporal_evidence") + evidence = evidence if isinstance(evidence, dict) else {} + raw_intervals = evidence.get("gap_other_intervals") + if not isinstance(raw_intervals, list): + raw_intervals = [] + onsets: set[float] = set() + for index, interval in enumerate(raw_intervals): + if not isinstance(interval, dict): + continue + onset = _optional_seconds( + interval.get("onset_seconds"), + field=f"gap_other_intervals[{index}].onset_seconds", + uid=uid, + ) + if onset is None: + continue + if ( + onset >= selected_release_seconds - 1e-7 + and onset <= behavior_anchor_seconds + 1e-7 + ): + onsets.add(onset) + return sorted(onsets) + + +def _int_competing_intervals( candidate: dict[str, Any], *, behavior_anchor_seconds: float, uid: str, -) -> tuple[float, float, int] | None: +) -> list[tuple[float, float]]: evidence = candidate.get("temporal_evidence") evidence = evidence if isinstance(evidence, dict) else {} raw_intervals = evidence.get("nearby_other_intervals") if not isinstance(raw_intervals, list): - return None + return [] eligible: list[tuple[float, float]] = [] for index, interval in enumerate(raw_intervals): if not isinstance(interval, dict): @@ -209,13 +279,7 @@ def _int_competing_interval( continue if onset < behavior_anchor_seconds and offset > onset: eligible.append((onset, offset)) - if not eligible: - return None - # A semantic INT review does not currently select one competing onset. - # The latest interval crossing the wearer offset is the deterministic, - # most proximal causal candidate. Multiplicity is surfaced for audit. - onset, offset = max(eligible, key=lambda item: (item[0], item[1])) - return onset, offset, len(eligible) + return sorted(set(eligible)) def load_opportunity_events( @@ -278,6 +342,38 @@ def load_opportunity_events( if selected_release is None: skipped["entry_positive_without_selected_release"] += 1 continue + selected_option = _entry_selected_option( + candidate, + review, + selected_release_seconds=selected_release, + uid=uid, + ) + if ( + selected_option is not None + and selected_option.get( + "wearer_active_between_release_and_onset" + ) is True + ): + skipped["entry_positive_post_outcome_wearer_activity"] += 1 + continue + competing_onsets = _entry_competing_onsets( + candidate, + selected_release_seconds=selected_release, + behavior_anchor_seconds=behavior, + uid=uid, + ) + option_reports_competitor = ( + selected_option is not None + and selected_option.get( + "other_onset_between_release_and_onset" + ) is True + ) + if option_reports_competitor and not competing_onsets: + skipped["entry_positive_competing_onset_unresolved"] += 1 + continue + if competing_onsets: + skipped["entry_positive_competed_before_behavior"] += 1 + continue anchor = selected_release resolution = behavior censor = behavior @@ -298,17 +394,21 @@ def load_opportunity_events( competing_intervals = 0 else: if positive: - interval = _int_competing_interval( + intervals = _int_competing_intervals( candidate, behavior_anchor_seconds=behavior, uid=uid, ) - if interval is None: + if not intervals: skipped["int_positive_without_crossing_other"] += 1 continue - anchor, censor, competing_intervals = interval + if len(intervals) > 1: + skipped["int_positive_ambiguous_competing_others"] += 1 + continue + anchor, censor = intervals[0] + competing_intervals = 1 resolution = behavior - basis = "latest_other_onset_crossing_wearer_offset" + basis = "unique_other_onset_crossing_wearer_offset" elif subtype_value in INT_HARD_NEGATIVES: if source in INT_MINED_SOURCES: anchor, censor = _candidate_region(candidate, uid=uid) @@ -316,17 +416,21 @@ def load_opportunity_events( competing_intervals = 1 basis = "mined_competing_other_onset" else: - interval = _int_competing_interval( + intervals = _int_competing_intervals( candidate, behavior_anchor_seconds=behavior, uid=uid, ) - if interval is None: + if not intervals: skipped["int_hn_without_competing_other"] += 1 continue - anchor, censor, competing_intervals = interval + if len(intervals) > 1: + skipped["int_hn_ambiguous_competing_others"] += 1 + continue + anchor, censor = intervals[0] + competing_intervals = 1 resolution = behavior - basis = "latest_other_onset_crossing_non_yield_offset" + basis = "unique_other_onset_crossing_non_yield_offset" else: skipped["int_unsupported_hard_negative"] += 1 continue @@ -731,8 +835,8 @@ def main() -> None: else calibration_diagnostics["num_provisional_llm"] ) payload = { - "schema_version": 1, - "protocol": "egospeak-opportunity-conditioned-floor-actions-v1", + "schema_version": 2, + "protocol": "egospeak-opportunity-conditioned-floor-actions-v2", "semantic_scope": ( "LLM-assisted opportunity labels with dense wearer-activity scores; " "this is provisional and no human review is implied." @@ -757,6 +861,13 @@ def main() -> None: "yield_max_seconds": yield_max, "entry_headline_hard_negatives": sorted(ENTRY_CORE_HARD_NEGATIVES), "entry_backchannel_policy": "separate diagnostic; excluded from headline", + "entry_competing_risk_policy": ( + "exclude selected-release positives with an other onset at or before " + "the wearer entry, or with intervening wearer activity" + ), + "int_multiple_competitor_policy": ( + "exclude until one competing onset is selected by review" + ), "matched_recalls": recalls, "review_diagnostics": review_diagnostics, "calibration_review_diagnostics": calibration_diagnostics, diff --git a/manuscript/main/main.tex b/manuscript/main/main.tex index 9474391..d6dc707 100644 --- a/manuscript/main/main.tex +++ b/manuscript/main/main.tex @@ -406,8 +406,8 @@ verify identical targets, causal validity, and evaluation.} \begin{table}[t] \caption{Confirmatory EasyCom results. Entry and INT false-positive rates use the opportunity-conditioned activity score at matched recall $R=0.8$; E/I AP -is the corresponding opportunity AP. $\dagger$ denotes provisional -LLM-assisted labels pending human review.} +is the corresponding opportunity AP. Event results will be populated only +after protocol-v2 reruns and human adjudication.} \label{tab:main} \centering \small @@ -417,7 +417,7 @@ LLM-assisted labels pending human review.} Method & 0.2-s mAP $\uparrow$ & Mean mAP $\uparrow$ & Entry FPR@$R=.8$ $\downarrow$ & INT FPR@$R=.8$ $\downarrow$ & E/I AP $\uparrow$\\ \hline -EgoSpeak MiniROADA A+V & 74.12 & 61.16 & \todo{--} & \todo{--} & $52.49/20.13^\dagger$\\ +EgoSpeak MiniROADA A+V & 74.12 & 61.16 & \todo{--} & \todo{--} & \todo{--/--}\\ Omni-3B frozen $G$ & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\ Omni-3B matched PEFT & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\ $G+$local audio & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \todo{--/--}\\ @@ -430,10 +430,9 @@ Omni-7B scaling control & \todo{--} & \todo{--} & \todo{--} & \todo{--} & \t \textbf{Dense forecasting.} The EgoSpeak baseline reaches 74.12\% 0.2-s mAP and 61.16\% mean mAP; its raw -wearer-onset/offset mean APs are 24.49/38.02\%. On provisional reviewed -opportunities, ENTRY AP is 52.49\% versus 32.73\% prevalence, whereas INT AP is -20.13\% versus 20.44\%; continuous wearer-score plots therefore do not alone -establish semantic floor-yield selectivity. +wearer-onset/offset mean APs are 24.49/38.02\%. Opportunity-conditioned values +remain pending a protocol-v2 rerun and human review; continuous wearer-score +plots do not alone establish semantic floor-yield selectivity. The proposed local AV adapter improves 0.2-s mAP from \todo{$X$ to $Y$} over the strongest matched baseline and improves mean mAP by \todo{$Z$ points}. diff --git a/results/easycom_egospeak_three_level_eval_2026-09-16/README.md b/results/easycom_egospeak_three_level_eval_2026-09-16/README.md index 562adfe..376a867 100644 --- a/results/easycom_egospeak_three_level_eval_2026-09-16/README.md +++ b/results/easycom_egospeak_three_level_eval_2026-09-16/README.md @@ -1,5 +1,11 @@ # EasyCom EgoSpeak three-level evaluation +> **Protocol erratum.** The dense and raw-boundary results in this record remain +> valid. The opportunity-conditioned ENTRY/INT values were produced by the +> superseded v1 loader, which could cross an intervening ENTRY competitor and +> selected the latest onset in ambiguous multi-competitor INT cases. Do not cite +> those opportunity values; rerun protocol v2 and create a new result record. + This record freezes the first Sessions 1--3 evaluation of the released EgoSpeak MiniROADA audio-visual checkpoint under three deliberately separate questions. The values below were produced on the remote experiment host; the @@ -15,7 +21,7 @@ prediction and per-event artifacts remain on that host and are not committed. - Raw boundary anticipation: threshold-free AP for wearer `OFF->ON` and `ON->OFF` changes. These are timing-derived diagnostics, not semantic EOT or INT claims. -- Opportunity-conditioned behavior: one causally censored activity-derived +- Opportunity-conditioned behavior: historical v1 activity-derived score per reviewed ENTRY or INT/YIELD opportunity. Labels are provisional two-pass LLM-assisted reviews and have not been human reviewed. - No test-set threshold is fitted for any AP reported here. Session 4 is needed @@ -34,15 +40,16 @@ mean AP, respectively. These lower values do not contradict the dense result: dense AP rewards correct future state over sustained regions, whereas a raw boundary event is counted once and has many harder non-boundary comparisons. -On the provisional opportunity labels, ENTRY AP is **52.487%** for 163 +The superseded v1 run reported ENTRY AP of **52.487%** for 163 positives and 335 hard negatives, versus 32.731% positive prevalence (+19.756 points; 1.60x prevalence). INT/YIELD AP is **20.135%** for 28 positives -and 109 hard negatives, essentially equal to its 20.438% prevalence. This is -consistent with the output contract: wearer probability can rank many entry -opportunities, but `1-p(wearer)` does not encode whether an interruption should -cause a yield. The ENTRY and INT loaders additionally exclude 81 and 5 -invalid/post-outcome cases; these exclusions require an audit before any final -paper claim. +and 109 hard negatives, essentially equal to its 20.438% prevalence. These +values are retained only for provenance and must be replaced by a v2 rerun. The +original pattern was consistent with the output contract: wearer probability +can rank many entry opportunities, but `1-p(wearer)` does not encode whether an +interruption should cause a yield. The ENTRY and INT loaders additionally +exclude 81 and 5 invalid/post-outcome cases; these exclusions require an audit +before any final paper claim. ## Relation to EgoSpeak Figure 7 @@ -73,4 +80,3 @@ Machine-readable curves and counts are in [`summary.json`](summary.json) and the compact comparison is in [`metrics.csv`](metrics.csv). The opportunity protocol and reproduction command are documented in [`docs/easycom_opportunity_event_evaluation.md`](../../docs/easycom_opportunity_event_evaluation.md). - diff --git a/results/easycom_egospeak_three_level_eval_2026-09-16/summary.json b/results/easycom_egospeak_three_level_eval_2026-09-16/summary.json index 37164a7..6d3d2fa 100644 --- a/results/easycom_egospeak_three_level_eval_2026-09-16/summary.json +++ b/results/easycom_egospeak_three_level_eval_2026-09-16/summary.json @@ -1,6 +1,8 @@ { "recorded_at": "2026-09-16", "source": "user-provided remote experiment output", + "opportunity_protocol_status": "superseded_v1_rerun_required", + "opportunity_protocol_erratum": "Dense and raw-boundary metrics remain valid; opportunity metrics must be rerun with egospeak-opportunity-conditioned-floor-actions-v2.", "dataset": "EasyCom", "sessions": ["Session_1", "Session_2", "Session_3"], "model": "EgoSpeak MiniROADA audio-visual released checkpoint", @@ -57,4 +59,3 @@ "threshold_free": true, "session4_required_for_reported_metrics": false } - diff --git a/results/easycom_omni_floor_screen_2026-09-16/README.md b/results/easycom_omni_floor_screen_2026-09-16/README.md index dae4fdb..7b1d554 100644 --- a/results/easycom_omni_floor_screen_2026-09-16/README.md +++ b/results/easycom_omni_floor_screen_2026-09-16/README.md @@ -1,5 +1,12 @@ # EasyCom Omni floor-event screening +> **Protocol erratum.** Dense and raw-boundary metrics remain valid. All +> opportunity-conditioned ENTRY/INT values below used the superseded v1 loader, +> which could cross an intervening ENTRY competitor and selected the latest +> onset in ambiguous multi-competitor INT cases. Retain them for provenance +> only; do not use them for model selection or manuscript claims before a v2 +> rerun. + This record freezes the first comparison of EgoSpeak, Omni `G+A`, Omni `G+A+pooled-V`, and the three-seed current-frame F1 Frame-set Attention screen on EasyCom Sessions 1--3. Values were produced on the remote experiment host @@ -11,9 +18,8 @@ CSVs remain on that host and are not committed here. 1. Dense future activity uses causal-valid framewise AP/mAP over ten horizons. 2. Raw wearer endpoint and aggregate-other-boundary AP are threshold-free timing diagnostics; they are not semantic EOT/INT labels. -3. Opportunity-conditioned ENTRY and INT/YIELD AP assigns one causally - censored activity-derived score to each provisional LLM-assisted reviewed - opportunity. +3. Opportunity-conditioned ENTRY and INT/YIELD AP is a historical v1 screen + that requires rerunning with the corrected v2 competing-risk protocol. All opportunity runs use identical denominators: 163 ENTRY positives and 335 hard negatives, with 81 invalid/post-outcome cases; and 28 INT positives and @@ -39,7 +45,7 @@ from EgoSpeak to G+A to G+A+pooled-V as follows: onset AP offset/yield-timing gain comes from G+A; pooled visual input adds almost nothing to raw transition timing. -## Interpretation +## Historical v1 interpretation - G+A strongly improves dense forecasting and generic wearer-offset/yield timing over EgoSpeak, but not reviewed semantic opportunities. In particular, @@ -77,4 +83,3 @@ nothing to raw transition timing. Exact aggregate values are stored in [`summary.json`](summary.json) and [`metrics.csv`](metrics.csv). The earlier EgoSpeak-only record is [`easycom_egospeak_three_level_eval_2026-09-16`](../easycom_egospeak_three_level_eval_2026-09-16/README.md). - diff --git a/results/easycom_omni_floor_screen_2026-09-16/summary.json b/results/easycom_omni_floor_screen_2026-09-16/summary.json index 3e9d6dc..a704fd9 100644 --- a/results/easycom_omni_floor_screen_2026-09-16/summary.json +++ b/results/easycom_omni_floor_screen_2026-09-16/summary.json @@ -1,6 +1,8 @@ { "recorded_at": "2026-09-16", "source": "user-provided remote experiment output", + "opportunity_protocol_status": "superseded_v1_rerun_required", + "opportunity_protocol_erratum": "Dense and raw-boundary metrics remain valid; opportunity metrics must be rerun with egospeak-opportunity-conditioned-floor-actions-v2.", "dataset": "EasyCom", "test_sessions": ["Session_1", "Session_2", "Session_3"], "scale": "percent", @@ -82,4 +84,3 @@ "threshold_free_metrics_only": true, "matched_pooled_control_available": false } - diff --git a/test_evaluate_egospeak_opportunity_events.py b/test_evaluate_egospeak_opportunity_events.py index fd74ee3..36dee2f 100644 --- a/test_evaluate_egospeak_opportunity_events.py +++ b/test_evaluate_egospeak_opportunity_events.py @@ -30,7 +30,10 @@ def reviewed_row( region_start: float, region_end: float, selected_release: float | None = None, + selected_release_uid: str | None = None, nearby: list[dict[str, object]] | None = None, + opportunity_options: list[dict[str, object]] | None = None, + gap_other_intervals: list[dict[str, object]] | None = None, candidate_source: str | None = None, ) -> dict[str, object]: event_type = None @@ -45,6 +48,7 @@ def reviewed_row( "human_verdict": verdict, "human_event_type": event_type, "human_subtype": subtype, + "selected_opportunity_uid": selected_release_uid, "selected_opportunity_seconds": selected_release, "boundary_correction_seconds": 0, }, @@ -59,6 +63,8 @@ def reviewed_row( "behavior_anchor_seconds": anchor, "wearer_offset_seconds": anchor, "nearby_other_intervals": nearby or [], + "opportunity_options": opportunity_options or [], + "gap_other_intervals": gap_other_intervals or [], }, }, } @@ -119,7 +125,7 @@ class OpportunityEventTests(unittest.TestCase): diagnostics["skipped"]["entry_backchannel_separate_diagnostic"], 1 ) - def test_loader_selects_latest_crossing_other_for_int(self) -> None: + def test_loader_excludes_ambiguous_crossing_others_for_int(self) -> None: nearby = [ { "onset_seconds": 1.0, @@ -145,11 +151,91 @@ class OpportunityEventTests(unittest.TestCase): ) ]) self.addCleanup(temporary.cleanup) - events, _ = load_opportunity_events([path], ["Session_4"]) - self.assertEqual(len(events), 1) - self.assertEqual(events[0].opportunity_anchor_seconds, 2.0) - self.assertEqual(events[0].censor_seconds, 3.8) - self.assertEqual(events[0].competing_intervals, 2) + events, diagnostics = load_opportunity_events([path], ["Session_4"]) + self.assertEqual(events, []) + self.assertEqual( + diagnostics["skipped"]["int_positive_ambiguous_competing_others"], + 1, + ) + + def test_loader_excludes_entry_positive_at_competing_other_onset(self) -> None: + rows = [ + reviewed_row( + "entry-competed", + family="entry", + verdict="POSITIVE", + subtype=None, + anchor=1.6, + region_start=0.0, + region_end=3.0, + selected_release=1.0, + selected_release_uid="release", + opportunity_options=[ + { + "uid": "release", + "anchor_seconds": 1.0, + "wearer_active_between_release_and_onset": False, + "other_onset_between_release_and_onset": True, + } + ], + gap_other_intervals=[ + {"uid": "competitor", "onset_seconds": 1.4} + ], + ), + reviewed_row( + "entry-simultaneous", + family="entry", + verdict="POSITIVE", + subtype=None, + anchor=4.0, + region_start=3.0, + region_end=5.0, + selected_release=3.2, + gap_other_intervals=[ + {"uid": "competitor", "onset_seconds": 4.0} + ], + ), + ] + temporary, path = self.write_reviews(rows) + self.addCleanup(temporary.cleanup) + events, diagnostics = load_opportunity_events([path], ["Session_4"]) + self.assertEqual(events, []) + self.assertEqual( + diagnostics["skipped"]["entry_positive_competed_before_behavior"], + 2, + ) + + def test_loader_excludes_entry_positive_after_prior_wearer_activity(self) -> None: + temporary, path = self.write_reviews([ + reviewed_row( + "entry-post-outcome", + family="entry", + verdict="POSITIVE", + subtype=None, + anchor=2.0, + region_start=0.0, + region_end=3.0, + selected_release=1.0, + selected_release_uid="release", + opportunity_options=[ + { + "uid": "release", + "anchor_seconds": 1.0, + "wearer_active_between_release_and_onset": True, + "other_onset_between_release_and_onset": False, + } + ], + ) + ]) + self.addCleanup(temporary.cleanup) + events, diagnostics = load_opportunity_events([path], ["Session_4"]) + self.assertEqual(events, []) + self.assertEqual( + diagnostics["skipped"][ + "entry_positive_post_outcome_wearer_activity" + ], + 1, + ) def test_entry_hn_requires_matching_independently_mined_source(self) -> None: rows = [ @@ -385,7 +471,7 @@ class OpportunityEventTests(unittest.TestCase): payload = json.loads(output.read_text(encoding="utf-8")) self.assertEqual( payload["protocol"], - "egospeak-opportunity-conditioned-floor-actions-v1", + "egospeak-opportunity-conditioned-floor-actions-v2", ) self.assertTrue((root / "summary_metrics.csv").is_file()) self.assertTrue((root / "summary_events.csv").is_file())