Match EasyCom oracle labels to EgoSpeak sampling

docs/easycom_oracle_information.md

@@ -10,8 +10,11 @@ This is an information upper-bound experiment, not a deployable input setting.
 ## Causal contract
 
 - The task grid remains 5 Hz. State row `t` has cutoff `(t+1)/5` seconds.
-- Oracle bin `t` summarizes transcript frames in
+- Oracle state `t` follows EgoSpeak's released-target conversion exactly:
-  `[t/5,(t+1)/5)` and is therefore completely observed at that cutoff.
+  source-frame markers `4, 8, 12, ...` are sampled at the corresponding
+  `0.2, 0.4, 0.6, ...` second cutoffs, with
+  `Start_Frame <= marker < End_Frame`. This is right-edge point sampling, not
+  an any-overlap reduction over a 200 ms bin.
 - Each person vector stores the last 2 seconds of binary activity followed by
   a visibility/availability mask. Activity is exposed only in bins where that
   participant was visible according to the provided boxes. No future bin,
@@ -19,8 +22,8 @@ This is an information upper-bound experiment, not a deployable input setting.
 - Only non-wearer participant IDs present in the latest provided-box frame
   strictly before the cutoff receive slots. IDs route annotations during
   preparation but are not exposed as embeddings to the head.
-- Transcript-derived wearer/others activity must match `target_perframe` by
+- Transcript-derived wearer/others activity must match `target_perframe`
-  default. Any mismatch fails closed instead of being silently tolerated.
+  exactly. Any mismatch fails closed; there is no tolerance flag.
 - Oracle sidecars use `easycom-oracle-person-history-v1` and are accepted only
   by the explicit `oracle-frame-set-attention` mode. Normal local-feature
   loading rejects them because they use speech labels.
@@ -65,8 +68,8 @@ dense AP/mAP screen. Existing compatible oracle sidecars are validated and
 reused, so interrupted preparation can resume. It deliberately does not run
 Sessions 1--3.
 
-If preparation reports any transcript/target mismatch, stop and inspect it.
+If preparation reports any transcript/target mismatch, stop and inspect it;
-Do not raise `--max-label-mismatch-rows` merely to make the run continue.
+the preparation command has no mismatch-tolerance option.
 
 ## Follow-up
 

local_turn_features.py

@@ -301,7 +301,7 @@ def _load_validated_payload(
     if oracle_analysis:
         if (
             dataset != "easycom"
-            or config.get("causal_policy") != "fully-observed-half-open-5hz-bins"
+            or config.get("causal_policy") != "egospeak-right-edge-current-state"
             or config.get("uses_speech_labels") is not True
             or config.get("oracle_analysis") is not True
         ):
@@ -441,13 +441,21 @@ def load_frame_set_features(
     if len(frame_times) and float(frame_times[-1]) > float(times[-1]) + 1e-8:
         raise ValueError(f"Person frame timeline extends beyond the reference: {path}")
 
-    # Match the sidecar's half-open policy: a frame stamped exactly at the
+    # Normal visual observations must be strictly earlier than the cutoff.
-    # state cutoff is not yet part of that state's observed prefix.
+    # Oracle speech state is defined and sampled at the cutoff itself, so its
-    row_to_frame = torch.searchsorted(frame_times, times, right=False) - 1
+    # explicit analysis-only format uses an inclusive current-state lookup.
+    row_to_frame = torch.searchsorted(
+        frame_times,
+        times,
+        right=oracle_analysis,
+    ) - 1
     chosen = torch.full(times.shape, float("nan"), dtype=torch.float64)
     has_frame = row_to_frame >= 0
     chosen[has_frame] = frame_times[row_to_frame[has_frame]]
-    if bool((chosen[has_frame] >= times[has_frame]).any()):
+    if oracle_analysis:
+        if bool((chosen[has_frame] > times[has_frame]).any()):
+            raise ValueError(f"Oracle person mapping selected future evidence: {path}")
+    elif bool((chosen[has_frame] >= times[has_frame]).any()):
         raise ValueError(f"Person frame mapping selected future visual evidence: {path}")
     previous_observation = None
     if track_max_gap_seconds is not None:

prepare_easycom_oracle_features.py

@@ -64,36 +64,49 @@ def participant_activity(
     *,
     num_steps: int,
     wearer_id: int,
-) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, int]]:
+) -> tuple[dict[int, torch.BoolTensor], torch.BoolTensor, dict[str, Any]]:
-    """Rasterize inclusive 20 Hz transcript spans into observed 5 Hz bins."""
+    """Sample transcript activity on EgoSpeak's 5 Hz right-edge grid.
+
+    EasyCom stores one-based ``Start_Frame``/``End_Frame`` markers. EgoSpeak's
+    released targets sample source-frame markers 4, 8, 12, ... and use
+    ``Start_Frame <= marker < End_Frame``. ``load_utterances`` represents the
+    corresponding interval as ``[Start_Frame - 1, End_Frame)`` on its
+    zero-origin dense timeline, hence the ``+1`` below. This is deliberately
+    point sampling, not an any-overlap reduction over a 200 ms bin.
+    """
 
     activity: dict[int, torch.BoolTensor] = {}
     wearer = torch.zeros(num_steps, dtype=torch.bool)
-    discarded_terminal_frames = 0
+    terminal_frames_after_last_sample = 0
     grid_frames = num_steps * FRAMES_PER_TASK_STEP
     for row in utterances:
-        dense_start = int(row.dense_start_frame)
+        # Recover the source annotation marker from the zero-origin interval.
+        dense_start = int(row.dense_start_frame) + 1
         dense_end = int(row.dense_end_frame_exclusive)
         if dense_end > grid_frames:
             overrun = dense_end - grid_frames
             if overrun > FRAMES_PER_TASK_STEP - 1:
                 raise ValueError(
-                    f"Transcript exceeds the discarded terminal sub-step by "
+                    f"Transcript exceeds the final 5 Hz sample by "
                     f"{overrun} frames: {row.uid}"
                 )
-            discarded_terminal_frames += max(
+            terminal_frames_after_last_sample += overrun
-                0, dense_end - max(dense_start, grid_frames)
-            )
-            dense_end = grid_frames
         if dense_start >= dense_end:
             continue
-        first = dense_start // FRAMES_PER_TASK_STEP
+
-        last = (dense_end - 1) // FRAMES_PER_TASK_STEP
+        # Sample number k (one-origin) observes marker 4*k and maps to row k-1.
-        if first < 0 or last < first or last >= num_steps:
+        # The exclusive stop follows directly from 4*k < dense_end.
+        first = (dense_start + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
+        stop = (dense_end + FRAMES_PER_TASK_STEP - 1) // FRAMES_PER_TASK_STEP - 1
+        first = max(0, first)
+        stop = min(num_steps, stop)
+        if first > num_steps or stop < 0:
             raise ValueError(
                 f"Transcript interval leaves target grid: {row.uid}, "
-                f"bins=[{first},{last}], target_steps={num_steps}"
+                f"rows=[{first},{stop}), target_steps={num_steps}"
             )
+        if first >= stop:
+            continue
         destination = (
             wearer
             if int(row.participant_id) == wearer_id
@@ -102,9 +115,12 @@ def participant_activity(
                 torch.zeros(num_steps, dtype=torch.bool),
             )
         )
-        destination[first : last + 1] = True
+        destination[first:stop] = True
     return activity, wearer, {
-        "discarded_terminal_transcript_frames": discarded_terminal_frames
+        "target_sampling_policy": "egospeak-right-edge-start-inclusive-end-exclusive",
+        "terminal_transcript_frames_after_last_5hz_sample": (
+            terminal_frames_after_last_sample
+        ),
     }
 
 
@@ -323,7 +339,6 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--delay-seconds", type=float, default=1.0)
     parser.add_argument("--camera-wearer-id", type=int, default=CAMERA_WEARER_ID)
     parser.add_argument("--shuffle-seed", type=int, default=2027)
-    parser.add_argument("--max-label-mismatch-rows", type=int, default=0)
     parser.add_argument("--expected-mode", default="open-stream-replay")
     parser.add_argument("--reuse-existing", action="store_true")
     parser.add_argument("--overwrite", action="store_true")
@@ -335,8 +350,6 @@ def main() -> None:
     args = parse_args()
     history_steps = exact_steps(args.history_seconds, "--history-seconds")
     delay_steps = exact_steps(args.delay_seconds, "--delay-seconds")
-    if args.max_label_mismatch_rows < 0:
-        raise ValueError("--max-label-mismatch-rows cannot be negative")
     sessions = [normalize_sample_name(value, "easycom") for value in args.sessions]
     unknown = sorted(set(sessions) - set(ALL_SESSIONS))
     if unknown:
@@ -405,12 +418,13 @@ def main() -> None:
                 | (all_other_activity != target_tensor[:, 2].bool())
             ).sum()
         )
-        if mismatch_union > args.max_label_mismatch_rows:
+        if mismatch_union:
             raise ValueError(
                 f"{session}: transcript/target activity differs on "
                 f"{mismatch_union} rows (wearer={wearer_mismatch}, "
-                f"others={other_mismatch}); inspect alignment or explicitly "
+                f"others={other_mismatch}) under the EgoSpeak right-edge "
-                "raise --max-label-mismatch-rows"
+                "sampling rule; inspect transcript ordering and dense-timeline "
+                "alignment. Oracle preparation never tolerates label mismatch."
             )
 
         visible_ids, source_records = visible_people_by_row(
@@ -501,7 +515,7 @@ def main() -> None:
             config = {
                 "oracle_analysis": True,
                 "uses_speech_labels": True,
-                "causal_policy": "fully-observed-half-open-5hz-bins",
+                "causal_policy": "egospeak-right-edge-current-state",
                 "oracle_condition": condition,
                 "oracle_scope": "currently-visible-nonwearer-participants",
                 "history_steps": history_steps,
@@ -531,10 +545,9 @@ def main() -> None:
                 "audio_valid": audio_valid,
                 "persons": {
                     "features": features,
-                    # Bin k covers [k/5,(k+1)/5); its feature is available at
+                    # This oracle state is sampled exactly at the matching
-                    # the matching state cutoff (k+1)/5.
+                    # reference cutoff and is available at that cutoff.
-                    "frame_times": torch.arange(len(times), dtype=torch.float64)
+                    "frame_times": times.clone(),
-                    * STEP_SECONDS,
                     "frame_offsets": frame_offsets,
                     "records": records,
                 },

test_easycom_oracle_features.py

@@ -23,27 +23,33 @@ class OracleFeatureTests(unittest.TestCase):
     def setUp(self) -> None:
         torch.set_num_threads(1)
 
-    def test_transcript_intervals_are_rasterized_into_half_open_bins(self) -> None:
+    def test_transcript_intervals_match_egospeak_right_edge_sampling(self) -> None:
         rows = [
             SimpleNamespace(
                 uid="a",
                 participant_id=3,
-                dense_start_frame=3,
+                # EasyCom annotation Start_Frame=1, End_Frame=8. Samples at
-                dense_end_frame_exclusive=5,
+                # source markers 4,8,... therefore include only row 0.
+                dense_start_frame=0,
+                dense_end_frame_exclusive=8,
             ),
             SimpleNamespace(
                 uid="b",
                 participant_id=2,
-                dense_start_frame=8,
+                # Start_Frame=5, End_Frame=13 includes markers 8 and 12.
-                dense_end_frame_exclusive=12,
+                dense_start_frame=4,
+                dense_end_frame_exclusive=13,
             ),
         ]
         activity, wearer, audit = participant_activity(
             rows, num_steps=4, wearer_id=2
         )
-        self.assertEqual(activity[3].tolist(), [True, True, False, False])
+        self.assertEqual(activity[3].tolist(), [True, False, False, False])
-        self.assertEqual(wearer.tolist(), [False, False, True, False])
+        self.assertEqual(wearer.tolist(), [False, True, True, False])
-        self.assertEqual(audit["discarded_terminal_transcript_frames"], 0)
+        self.assertEqual(
+            audit["target_sampling_policy"],
+            "egospeak-right-edge-start-inclusive-end-exclusive",
+        )
 
     def test_history_at_row_does_not_read_later_activity(self) -> None:
         first = torch.tensor([0, 1, 0, 0, 0], dtype=torch.bool)
@@ -152,7 +158,7 @@ class OracleFeatureTests(unittest.TestCase):
             "config": {
                 "oracle_analysis": True,
                 "uses_speech_labels": True,
-                "causal_policy": "fully-observed-half-open-5hz-bins",
+                "causal_policy": "egospeak-right-edge-current-state",
                 "oracle_condition": "per-person",
                 "visual_dim": 6,
                 "audio_dim": 5,
@@ -179,6 +185,8 @@ class OracleFeatureTests(unittest.TestCase):
         self.assertEqual(audio.shape, (n, 6))
         self.assertEqual(people.dimension, 6)
         self.assertEqual(contract["format"], ORACLE_FORMAT)
+        self.assertTrue(torch.equal(people.row_to_frame, torch.arange(n)))
+        self.assertTrue(torch.allclose(people.chosen_frame_times, times))
         head = TurnPredictionHead(
             hidden_size=8,
             num_horizons=2,