Fix Ego4D target compatibility in local baselines

docs/ego4d_local_baselines.md

@@ -291,6 +291,13 @@ python run_ego4d_local_baselines.py "${EXP_ARGS[@]}" --seeds 1 2
 如果原 stacked baseline 使用 fixed-mean，可在四组统一加 `--layer-fusion fixed-mean`。
 不要拿不同 epoch/layer-fusion 的旧 baseline 直接计算新增分支的增量。
 
+Ego4D target 由 EgoSpeak/LSTR 的 `MCE` 当作 binary/multi-positive 权重直接读取；
+因此代码对 `dataset=ego4d` 保留其原始二值行，不强行套用 EasyCom 的四状态约束，
+也绝不把 `[0,0,0]` 等行擅自改成 `[1,0,0]` background。预检会汇总并打印所有
+非 canonical pattern，便于核对数据。EasyCom 仍严格限制为四个 activity states。
+当前四组实验关闭 transition objective，因此不会从非 canonical 行伪造 transition
+标签；如果以后启用 transition auxiliary，需要先定义语义和显式 valid mask。
+
 预检不会一次把 433 条全局特征全部装进 RAM；实际训练仍沿用原训练器，会加载整个 train split。
 last4 特征可能需要数十 GB 主机内存。多卡并行 head 训练也会复制主机内存，不是只看 GPU 是否空闲。
 

easycom_head.py

@@ -278,8 +278,18 @@ def _validate_loss_inputs(
         )
 
 
-def _validated_binary_activity_targets(targets: torch.Tensor) -> torch.Tensor:
+def _validated_binary_activity_targets(
-    """Validate the four legal EasyCom/EgoSpeak three-label activity rows."""
+    targets: torch.Tensor,
+    *,
+    allow_noncanonical_binary: bool = False,
+) -> torch.Tensor:
+    """Validate three-label activity rows.
+
+    EasyCom uses four explicit activity states.  Ego4D's released target files
+    are consumed by EgoSpeak's MCE objective as binary/multi-positive weights,
+    so the compatibility path preserves any binary row instead of silently
+    rewriting it to an EasyCom state.
+    """
 
     if targets.ndim < 1 or targets.shape[-1] != NUM_CLASSES:
         raise ValueError(
@@ -291,11 +301,22 @@ def _validated_binary_activity_targets(targets: torch.Tensor) -> torch.Tensor:
     rounded = targets.round()
     if not torch.allclose(targets, rounded, atol=1e-6, rtol=0.0):
         raise ValueError("Activity targets must contain only binary 0/1 values")
-    expected_background = (rounded[..., 1:].sum(dim=-1) == 0).to(rounded.dtype)
+    expected_background = rounded[..., 1:].sum(dim=-1) == 0
-    if not torch.equal(rounded[..., 0], expected_background):
+    valid = rounded[..., 0].bool() == expected_background
+    if allow_noncanonical_binary:
+        return rounded
+    if not bool(valid.all()):
+        invalid = torch.unique(rounded[~valid].reshape(-1, NUM_CLASSES), dim=0)
+        invalid_rows = ", ".join(
+            str([int(value) for value in row.tolist()])
+            for row in invalid[:8]
+        )
+        if len(invalid) > 8:
+            invalid_rows += ", ..."
         raise ValueError(
             "Activity targets must be one of [1,0,0], [0,1,0], "
-            "[0,0,1], or [0,1,1]"
+            "[0,0,1], or [0,1,1]; observed invalid "
+            f"row(s): {invalid_rows}"
         )
     return rounded
 
@@ -799,7 +820,13 @@ def load_session_data(
             f"{target_path}"
         )
     targets = torch.from_numpy(np.asarray(target_array, dtype=np.float32))
-    _validated_binary_activity_targets(targets)
+    try:
+        _validated_binary_activity_targets(
+            targets,
+            allow_noncanonical_binary=dataset == "ego4d",
+        )
+    except ValueError as error:
+        raise ValueError(f"{error}: {target_path}") from error
     if len(label_indices) and int(label_indices[-1]) >= len(targets):
         raise ValueError(
             f"Last label index {int(label_indices[-1])} exceeds target length "
@@ -823,6 +850,20 @@ def load_session_data(
             "video_sampling_grid",
         )
     }
+    expected_background = targets[:, 1:].sum(dim=-1) == 0
+    canonical_rows = targets[:, 0].bool() == expected_background
+    unique_rows, row_counts = torch.unique(
+        targets.to(dtype=torch.int64),
+        dim=0,
+        return_counts=True,
+    )
+    metadata["num_noncanonical_target_rows"] = int(
+        (~canonical_rows).sum().item()
+    )
+    metadata["target_row_pattern_counts"] = {
+        "".join(str(int(value)) for value in row.tolist()): int(count)
+        for row, count in zip(unique_rows, row_counts)
+    }
     local = None
     if local_feature_root is not None:
         from local_turn_features import load_local_features

run_ego4d_local_baselines.py

@@ -93,6 +93,8 @@ def preflight(args) -> str:
     shape, contract = None, None
     counts = {"visual": [0, 0], "audio": [0, 0]}
     total = 0
+    noncanonical_target_rows = 0
+    target_row_pattern_counts: dict[str, int] = {}
     for index, name in enumerate(names, 1):
         data = load_session_data(
             args.feature_root, args.target_root, name, dataset="ego4d",
@@ -117,6 +119,15 @@ def preflight(args) -> str:
                 counts[modality][1] += len(data.states)
                 offset += width + 1
         total += len(data.states)
+        noncanonical_target_rows += int(
+            data.feature_metadata.get("num_noncanonical_target_rows", 0)
+        )
+        for pattern, count in data.feature_metadata.get(
+            "target_row_pattern_counts", {}
+        ).items():
+            target_row_pattern_counts[pattern] = (
+                target_row_pattern_counts.get(pattern, 0) + int(count)
+            )
         del data  # Validate one clip at a time, including all test clips.
         if not args.no_progress and (index == 1 or index % 25 == 0 or index == len(names)):
             print(f"preflight: {index}/{len(names)} clips", flush=True)
@@ -128,6 +139,19 @@ def preflight(args) -> str:
     if fusion != "single" and len(shape) != 2:
         raise ValueError(f"--layer-fusion {fusion} requires stacked [T,L,D] global states")
     print(f"Validated {total} states; global shape per row={shape}; fusion={fusion}", flush=True)
+    if noncanonical_target_rows:
+        canonical_patterns = {"100", "010", "001", "011"}
+        details = ", ".join(
+            f"[{','.join(pattern)}]={count}"
+            for pattern, count in sorted(target_row_pattern_counts.items())
+            if pattern not in canonical_patterns
+        )
+        print(
+            "Ego4D binary target rows outside the EasyCom four-state "
+            f"constraint: {noncanonical_target_rows}/{total} ({details}); "
+            "preserved exactly for the official LSTR MCE objective.",
+            flush=True,
+        )
     for modality, (valid, count) in counts.items():
         if count:
             note = "; provided boxes, not an online detector" if modality == "visual" else ""

test_ego4d_local_workflow.py

@@ -181,6 +181,30 @@ class Ego4DLocalWorkflowTests(unittest.TestCase):
             if name != "baseline_seed0":
                 self.assertEqual(train[train.index("--local-modalities") + 1], name.split("_")[0])
 
+    def test_ego4d_noncanonical_binary_rows_are_preserved(self):
+        target_path = self.root / "targets" / f"{self.names[0]}.npy"
+        targets = np.load(target_path)
+        targets[3] = 0
+        targets[4] = [1, 1, 0]
+        np.save(target_path, targets)
+
+        data = load_session_data(
+            self.root / "features",
+            self.root / "targets",
+            self.names[0],
+            dataset="ego4d",
+            expected_mode="open-stream-replay",
+        )
+        self.assertTrue(torch.equal(data.targets[3], torch.zeros(3)))
+        self.assertTrue(
+            torch.equal(data.targets[4], torch.tensor([1.0, 1.0, 0.0]))
+        )
+        self.assertEqual(data.feature_metadata["num_noncanonical_target_rows"], 2)
+        output = io.StringIO()
+        with contextlib.redirect_stdout(output):
+            runner.preflight(runner.parse_args(self.runner_args(["--variants", "baseline"])))
+        self.assertIn("constraint: 2/60 ([0,0,0]=1, [1,1,0]=1)", output.getvalue())
+
     def test_family_preflight_catches_missing_test_sidecar_before_training(self):
         (self.root / "local" / f"{self.names[1]}.pt").unlink()
         with patch("run_ego4d_local_baselines.subprocess.run") as run:
@@ -219,6 +243,13 @@ class Ego4DLocalWorkflowTests(unittest.TestCase):
         self.assertFalse((self.root / "checkpoints").exists())
 
     def test_full_cpu_family_round_trip_and_summary(self):
+        # EgoSpeak's LSTR MCE consumes the released Ego4D rows as binary
+        # weights.  Preserve noncanonical rows, and do not derive transitions
+        # when that auxiliary objective is disabled.
+        target_path = self.root / "targets" / f"{self.names[0]}.npy"
+        targets = np.load(target_path)
+        targets[3] = 0
+        np.save(target_path, targets)
         self.split_feature_layout()
         env = dict(os.environ, OMP_NUM_THREADS="1", MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1")
         completed = subprocess.run([sys.executable, str(runner.REPO / "run_ego4d_local_baselines.py"),

train_turn_head.py

@@ -224,6 +224,7 @@ def _load_examples(
     local_feature_root: Path | None = None,
     local_modalities: str = "av",
     local_contract: dict[str, Any] | None = None,
+    compute_transition_targets: bool = True,
 ) -> tuple[
     torch.Tensor,
     torch.Tensor,
@@ -259,11 +260,19 @@ def _load_examples(
             if data.local_features is not None
             else torch.empty(len(states), 0)
         )
-        activity_sequence = torch.cat(
+        if compute_transition_targets:
-            (data.targets[current_indices, None, :], targets),
+            activity_sequence = torch.cat(
-            dim=1,
+                (data.targets[current_indices, None, :], targets),
-        )
+                dim=1,
-        transition_targets = transition_targets_from_activity(activity_sequence)
+            )
+            transition_targets = transition_targets_from_activity(
+                activity_sequence
+            )
+        else:
+            transition_targets = torch.zeros(
+                targets.shape[:-1],
+                dtype=targets.dtype,
+            )
         dense_example_count = len(states)
         if selected_indices is not None:
             if name not in selected_indices:
@@ -605,6 +614,7 @@ def main() -> None:
         sparse_anchor_indices,
         args.local_feature_root,
         args.local_modalities,
+        compute_transition_targets=args.transition_objective != "none",
     )
     local_contract = feature_metadata[0].get("local_contract")
     if not len(states):
@@ -613,11 +623,15 @@ def main() -> None:
         f"training examples: {len(states)}, feature_dim={hidden_size}, "
         f"input_layers={num_input_layers}"
     )
-    print(
+    if args.transition_objective == "none":
-        "transition targets: "
+        print("transition targets: disabled (not derived or loaded)")
-        f"{int(transition_targets.sum().item())}/{transition_targets.numel()} "
+    else:
-        f"positive ({float(transition_targets.mean()) * 100:.3f}%)"
+        print(
-    )
+            "transition targets: "
+            f"{int(transition_targets.sum().item())}/"
+            f"{transition_targets.numel()} positive "
+            f"({float(transition_targets.mean()) * 100:.3f}%)"
+        )
     if num_input_layers > 1 and args.layer_fusion == "single":
         raise ValueError(
             f"Features contain {num_input_layers} stacked layers; pass "
@@ -650,6 +664,7 @@ def main() -> None:
             args.local_feature_root,
             args.local_modalities,
             local_contract,
+            compute_transition_targets=args.transition_objective != "none",
         )
         if dev_hidden_size != hidden_size:
             raise ValueError(