From 02e83c0c119740f4b7472e96a758181ec86df721 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Wed, 16 Sep 2026 03:56:33 +0800 Subject: [PATCH] Record Omni floor-event screening results --- PROJECT_CONTEXT.md | 11 ++- docs/easycom_opportunity_event_evaluation.md | 9 ++ ...peaker_floor_experiment_plan_2026-08-30.md | 7 ++ results/README.md | 1 + .../README.md | 80 +++++++++++++++++ .../metrics.csv | 32 +++++++ .../summary.json | 85 +++++++++++++++++++ 7 files changed, 223 insertions(+), 2 deletions(-) create mode 100644 results/easycom_omni_floor_screen_2026-09-16/README.md create mode 100644 results/easycom_omni_floor_screen_2026-09-16/metrics.csv create mode 100644 results/easycom_omni_floor_screen_2026-09-16/summary.json diff --git a/PROJECT_CONTEXT.md b/PROJECT_CONTEXT.md index 234818c..ed3e0d1 100644 --- a/PROJECT_CONTEXT.md +++ b/PROJECT_CONTEXT.md @@ -72,8 +72,15 @@ result. causal person set. On the existing three-seed, four-epoch test screen it obtained `81.77745 +/- 0.15638` 0.2-second mAP and `64.95099 +/- 0.09451` ten-horizon mean mAP. This is stable, but is not a - matched improvement over the historical no-dev pooled result; correct/wrong - visual controls remain missing. + matched improvement over the historical no-dev pooled result. Its three-seed + boundary response/yield AP is `28.423 +/- 0.480` / `48.641 +/- 0.391`; on + provisional reviewed opportunities its ENTRY/INT AP is + `41.561 +/- 0.581` / `22.991 +/- 0.202`. These values are respectively + `-4.461/-1.275` points below the historical pooled-V seed-0 result, so the + current F1 screen does not support the hoped-for semantic-selectivity gain. + A same-split three-seed pooled control and correct/wrong visual controls + remain missing. See the immutable + [`screening record`](results/easycom_omni_floor_screen_2026-09-16/README.md). ### Measured local-fusion screens diff --git a/docs/easycom_opportunity_event_evaluation.md b/docs/easycom_opportunity_event_evaluation.md index fbbb681..9f8b09b 100644 --- a/docs/easycom_opportunity_event_evaluation.md +++ b/docs/easycom_opportunity_event_evaluation.md @@ -67,3 +67,12 @@ or post-outcome. These are provisional LLM-assisted labels, not human-reviewed confirmatory results. Dense, raw-boundary, and opportunity metrics and their Figure 7 interpretation are frozen together in the [`2026-09-16 result record`](../results/easycom_egospeak_three_level_eval_2026-09-16/README.md). + +The expanded Omni screen finds no opportunity-ranking gain from the current +F1 Frame-set Attention model: its three-seed ENTRY/INT AP is +41.561%/22.991%, below the historical pooled-V seed-0 values +46.022%/24.266%. Because F1 and that pooled run use different train/dev model- +selection protocols, this is a stable negative screen rather than a matched +confirmatory comparison. Dense, boundary, per-seed, and provenance details are +in the +[`Omni floor screening record`](../results/easycom_omni_floor_screen_2026-09-16/README.md). diff --git a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md index e94ab4b..50505a9 100644 --- a/docs/omni_speaker_floor_experiment_plan_2026-08-30.md +++ b/docs/omni_speaker_floor_experiment_plan_2026-08-30.md @@ -168,6 +168,13 @@ forecasting 的定性解释，不能代替 boundary 或 semantic opportunity 评 曲线和限制见 [`three-level baseline record`](../results/easycom_egospeak_three_level_eval_2026-09-16/README.md)。 +当前 F1 current-frame person-set screen 的三种子结果同样没有支持 semantic floor +selectivity：ENTRY/INT opportunity AP 为 `41.561 +/- 0.581` / `22.991 +/- 0.202`， +低于历史 pooled-V seed-0 的 `46.022/24.266`；dense mean mAP 也为 +`64.951 +/- 0.095`，没有超过历史 pooled-V 的 `65.400`。由于两者 train/dev +选择协议不同，这只能判为稳定的负向 screening，正式归因前必须补同协议三种子 pooled +control。详见 [`Omni floor screening record`](../results/easycom_omni_floor_screen_2026-09-16/README.md)。 + ### P3：第一优先方法——匿名逐人时序旁路 先做易解释、能在 8×L20 上完成的旁路，而不是直接修改整个 Thinker。 diff --git a/results/README.md b/results/README.md index 7a91a83..e68b606 100644 --- a/results/README.md +++ b/results/README.md @@ -6,6 +6,7 @@ readers to the right comparison. | Result record | Question answered | |---|---| +| [`easycom_omni_floor_screen_2026-09-16/`](easycom_omni_floor_screen_2026-09-16/README.md) | Whether G+A, pooled-V, or current-frame F1 improve raw boundaries and provisional semantic opportunities over EgoSpeak | | [`easycom_egospeak_three_level_eval_2026-09-16/`](easycom_egospeak_three_level_eval_2026-09-16/README.md) | How dense forecasting, raw boundary anticipation, and provisional opportunity-conditioned ENTRY/YIELD differ for the EgoSpeak baseline | | [easycom_oracle_pooled_wearer_audit_2026-09-10/](easycom_oracle_pooled_wearer_audit_2026-09-10/README.md) | Fixed-four-epoch three-seed oracle test results and the limited wearer gains of historical pooled AV | | [`easycom_open_stream_vs_causal_prefix_2026-08-19/`](easycom_open_stream_vs_causal_prefix_2026-08-19/README.md) | How prepared-media open-stream and causal-prefix features compare | diff --git a/results/easycom_omni_floor_screen_2026-09-16/README.md b/results/easycom_omni_floor_screen_2026-09-16/README.md new file mode 100644 index 0000000..dae4fdb --- /dev/null +++ b/results/easycom_omni_floor_screen_2026-09-16/README.md @@ -0,0 +1,80 @@ +# EasyCom Omni floor-event screening + +This record freezes the first comparison of EgoSpeak, Omni `G+A`, Omni +`G+A+pooled-V`, and the three-seed current-frame F1 Frame-set Attention screen +on EasyCom Sessions 1--3. Values were produced on the remote experiment host +and supplied by the user on 2026-09-16. Generated predictions and per-event +CSVs remain on that host and are not committed here. + +## Evaluated layers + +1. Dense future activity uses causal-valid framewise AP/mAP over ten horizons. +2. Raw wearer endpoint and aggregate-other-boundary AP are threshold-free + timing diagnostics; they are not semantic EOT/INT labels. +3. Opportunity-conditioned ENTRY and INT/YIELD AP assigns one causally + censored activity-derived score to each provisional LLM-assisted reviewed + opportunity. + +All opportunity runs use identical denominators: 163 ENTRY positives and 335 +hard negatives, with 81 invalid/post-outcome cases; and 28 INT positives and +109 hard negatives, with 5 invalid/post-outcome cases. Positive prevalence is +32.731% for ENTRY and 20.438% for INT. + +## Main results + +| Model | 0.2-s mAP | Mean mAP | Boundary response AP | Boundary yield AP | ENTRY AP | INT/YIELD AP | +|---|---:|---:|---:|---:|---:|---:| +| EgoSpeak | 74.204 | 61.193 | 29.989 | 41.078 | 52.487 | 20.135 | +| Omni G+A, seed 0 | 81.723 | 65.109 | 28.840 | 47.873 | 43.886 | 21.535 | +| Omni G+A+pooled-V, seed 0 | 82.027 | 65.400 | 29.601 | 48.344 | 46.022 | 24.266 | +| F1, three-seed mean | 81.777 | 64.951 | 28.423 | 48.641 | 41.561 | 22.991 | + +F1 standard deviations are 0.156 points for 0.2-s mAP, 0.095 for mean mAP, +0.480 for boundary response AP, 0.391 for boundary yield AP, 0.581 for ENTRY +AP, and 0.202 for INT/YIELD AP. + +The raw endpoint diagnostic, available for the three seed-0 baselines, changes +from EgoSpeak to G+A to G+A+pooled-V as follows: onset AP +24.491/25.960/26.019 and offset AP 38.024/45.251/45.371. Thus most generic +offset/yield-timing gain comes from G+A; pooled visual input adds almost +nothing to raw transition timing. + +## Interpretation + +- G+A strongly improves dense forecasting and generic wearer-offset/yield + timing over EgoSpeak, but not reviewed semantic opportunities. In particular, + its INT AP remains close to prevalence and its ENTRY AP is lower than + EgoSpeak's. +- Adding pooled-V over G+A gives only +0.304/+0.291 points in 0.2-s/mean mAP, + +0.059/+0.120 in raw onset/offset AP, and +0.761/+0.471 in boundary + response/yield AP. The larger but still preliminary changes are +2.135 ENTRY + AP and +2.732 INT AP. +- F1 is stable across seeds but does not improve on the historical pooled-V + run: its mean changes are -0.250/-0.449 points in 0.2-s/mean mAP, -1.178 in + boundary response AP, +0.297 in boundary yield AP, -4.461 in ENTRY AP, and + -1.275 in INT AP. The hoped-for pattern of unchanged dense/timing performance + with improved semantic opportunity ranking is absent. +- Current F1 attends to a current anonymous visual person set using G+A. It has + no per-person temporal history, person-conditioned audio, or verified + identity continuity. This screen therefore does not support promoting F1 as + the paper's main floor-reasoning method. + +## Hard limitations + +- The opportunity labels are provisional two-pass LLM-assisted annotations, + not human-reviewed confirmatory labels. +- The old G+A and pooled-V runs are seed-0 fixed-four-epoch models trained on + Sessions 4--12 without development selection. F1 trains on Sessions 4--11 + and uses Session 12 for checkpoint selection. Their comparison is historical, + not a protocol-matched attribution. +- F1 needs a three-seed pooled-V control trained and selected with exactly the + same train/dev protocol before a confirmatory conclusion. +- All matched-recall trigger/FPR values produced in these exploratory commands + selected thresholds on Sessions 1--3 and are excluded from this record. +- Aggregate `others` activity cannot establish speaker identity, addressee, or + semantic interruption intent. + +Exact aggregate values are stored in [`summary.json`](summary.json) and +[`metrics.csv`](metrics.csv). The earlier EgoSpeak-only record is +[`easycom_egospeak_three_level_eval_2026-09-16`](../easycom_egospeak_three_level_eval_2026-09-16/README.md). + diff --git a/results/easycom_omni_floor_screen_2026-09-16/metrics.csv b/results/easycom_omni_floor_screen_2026-09-16/metrics.csv new file mode 100644 index 0000000..ff12ede --- /dev/null +++ b/results/easycom_omni_floor_screen_2026-09-16/metrics.csv @@ -0,0 +1,32 @@ +model,seeds,metric,value_percent,sample_std_percent +egospeak,1,dense_0.2s_mAP,74.204, +egospeak,1,dense_mean_mAP,61.193, +egospeak,1,raw_onset_mean_AP,24.491, +egospeak,1,raw_offset_mean_AP,38.024, +egospeak,1,boundary_response_mean_AP,29.989, +egospeak,1,boundary_yield_mean_AP,41.078, +egospeak,1,ENTRY_opportunity_AP,52.4867, +egospeak,1,INT_YIELD_opportunity_AP,20.1349, +omni_ga,1,dense_0.2s_mAP,81.72257, +omni_ga,1,dense_mean_mAP,65.10895, +omni_ga,1,raw_onset_mean_AP,25.960, +omni_ga,1,raw_offset_mean_AP,45.251, +omni_ga,1,boundary_response_mean_AP,28.840, +omni_ga,1,boundary_yield_mean_AP,47.873, +omni_ga,1,ENTRY_opportunity_AP,43.8864, +omni_ga,1,INT_YIELD_opportunity_AP,21.5345, +omni_gav,1,dense_0.2s_mAP,82.02661, +omni_gav,1,dense_mean_mAP,65.39978, +omni_gav,1,raw_onset_mean_AP,26.019, +omni_gav,1,raw_offset_mean_AP,45.371, +omni_gav,1,boundary_response_mean_AP,29.601, +omni_gav,1,boundary_yield_mean_AP,48.344, +omni_gav,1,ENTRY_opportunity_AP,46.0215, +omni_gav,1,INT_YIELD_opportunity_AP,24.2662, +f1,3,dense_0.2s_mAP,81.77745,0.15638 +f1,3,dense_mean_mAP,64.95099,0.09451 +f1,3,boundary_response_mean_AP,28.4227,0.4796 +f1,3,boundary_yield_mean_AP,48.6407,0.3914 +f1,3,ENTRY_opportunity_AP,41.5607,0.5805 +f1,3,INT_YIELD_opportunity_AP,22.9911,0.2015 + diff --git a/results/easycom_omni_floor_screen_2026-09-16/summary.json b/results/easycom_omni_floor_screen_2026-09-16/summary.json new file mode 100644 index 0000000..3e9d6dc --- /dev/null +++ b/results/easycom_omni_floor_screen_2026-09-16/summary.json @@ -0,0 +1,85 @@ +{ + "recorded_at": "2026-09-16", + "source": "user-provided remote experiment output", + "dataset": "EasyCom", + "test_sessions": ["Session_1", "Session_2", "Session_3"], + "scale": "percent", + "opportunity_counts": { + "entry": {"positive": 163, "hard_negative": 335, "invalid_or_post_outcome": 81, "prevalence": 32.7309}, + "int_yield": {"positive": 28, "hard_negative": 109, "invalid_or_post_outcome": 5, "prevalence": 20.4380} + }, + "models": { + "egospeak": { + "dense_0.2s_map": 74.204, + "dense_mean_map": 61.193, + "raw_onset_mean_ap": 24.491, + "raw_offset_mean_ap": 38.024, + "boundary_response_mean_ap": 29.989, + "boundary_yield_mean_ap": 41.078, + "entry_opportunity_ap": 52.4867, + "int_yield_opportunity_ap": 20.1349 + }, + "omni_ga_seed0": { + "dense_0.2s_map": 81.72257, + "dense_mean_map": 65.10895, + "raw_onset_mean_ap": 25.960, + "raw_offset_mean_ap": 45.251, + "boundary_response_mean_ap": 28.840, + "boundary_yield_mean_ap": 47.873, + "entry_opportunity_ap": 43.8864, + "int_yield_opportunity_ap": 21.5345 + }, + "omni_gav_seed0": { + "dense_0.2s_map": 82.02661, + "dense_mean_map": 65.39978, + "raw_onset_mean_ap": 26.019, + "raw_offset_mean_ap": 45.371, + "boundary_response_mean_ap": 29.601, + "boundary_yield_mean_ap": 48.344, + "entry_opportunity_ap": 46.0215, + "int_yield_opportunity_ap": 24.2662 + }, + "f1_seed0": { + "dense_0.2s_map": 81.87817, + "dense_mean_map": 65.03482, + "boundary_response_mean_ap": 28.244, + "boundary_yield_mean_ap": 48.566, + "entry_opportunity_ap": 41.9125, + "int_yield_opportunity_ap": 22.8024 + }, + "f1_seed1": { + "dense_0.2s_map": 81.59730, + "dense_mean_map": 64.96959, + "boundary_response_mean_ap": 28.966, + "boundary_yield_mean_ap": 48.292, + "entry_opportunity_ap": 41.8788, + "int_yield_opportunity_ap": 22.9676 + }, + "f1_seed2": { + "dense_0.2s_map": 81.85689, + "dense_mean_map": 64.84857, + "boundary_response_mean_ap": 28.058, + "boundary_yield_mean_ap": 49.064, + "entry_opportunity_ap": 40.8907, + "int_yield_opportunity_ap": 23.2033 + }, + "f1_three_seed": { + "dense_0.2s_map_mean": 81.77745, + "dense_0.2s_map_sample_std": 0.15638, + "dense_mean_map_mean": 64.95099, + "dense_mean_map_sample_std": 0.09451, + "boundary_response_mean_ap_mean": 28.4227, + "boundary_response_mean_ap_sample_std": 0.4796, + "boundary_yield_mean_ap_mean": 48.6407, + "boundary_yield_mean_ap_sample_std": 0.3914, + "entry_opportunity_ap_mean": 41.5607, + "entry_opportunity_ap_sample_std": 0.5805, + "int_yield_opportunity_ap_mean": 22.9911, + "int_yield_opportunity_ap_sample_std": 0.2015 + } + }, + "labels": "provisional two-pass LLM-assisted; not human reviewed", + "threshold_free_metrics_only": true, + "matched_pooled_control_available": false +} +