Add Ego4D local AV ablation workflow

summarize_local_baselines.py

@@ -11,15 +11,27 @@ def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--root", type=Path, default=Path("results/easycom_local_b1"))
     parser.add_argument("--protocol", choices=("causal_valid_only", "egospeak_full_timeline"), default="causal_valid_only")
+    parser.add_argument("--dataset", choices=("easycom", "ego4d"), help="Reject results from a different dataset")
+    parser.add_argument("--paired-deltas", action="store_true", help="Compare matching seeds to baseline, plus AV minus audio")
     args = parser.parse_args()
     files = sorted(args.root.glob("*/metrics.json"))
     if not files:
         raise SystemExit(f"No */metrics.json in {args.root}")
     print(f"Protocol: {args.protocol}; all AP/mAP numbers are percentages")
     print(f"{'run':<23} {'0.2s mAP':>11} {'BG AP':>11} {'wearer AP':>11} {'others AP':>11} {'mean mAP':>11}")
-    groups = {}
+    groups, by_seed = {}, {}
+    comparison_signature = None
     for path in files:
         metrics = json.loads(path.read_text())
+        if args.dataset is not None and metrics.get("dataset") != args.dataset:
+            raise ValueError(f"Dataset mismatch in {path}: {metrics.get('dataset')}")
+        if args.paired_deltas:
+            signature = (metrics.get("dataset"), metrics.get("sessions"), metrics.get("checkpoint_epoch"),
+                         metrics.get("checkpoint_train_sessions"), metrics.get("checkpoint_dev_sessions"),
+                         metrics.get("loss"), metrics.get("layer_fusion"))
+            if comparison_signature is not None and signature != comparison_signature:
+                raise ValueError(f"Paired runs have different splits/epochs/loss/fusion: {path}")
+            comparison_signature = signature
         score = metrics[args.protocol]
         first = score["horizons"][0]
         ap = first["class_ap"]
@@ -29,6 +41,9 @@ def main():
         print(f"{name:<23}" + "".join(f" {v:11.5f}" for v in values))
         group = re.sub(r"_seed\d+$", "", name)
         groups.setdefault(group, []).append(values)
+        match = re.fullmatch(r"(.+)_seed(\d+)", name)
+        if match:
+            by_seed.setdefault(match[1], {})[int(match[2])] = values
     print("\nMean ± sample standard deviation (AP/mAP percentage points)")
     for group, rows in sorted(groups.items()):
         stats = []
@@ -36,6 +51,21 @@ def main():
             spread = f"{statistics.stdev(column):.5f}" if len(column) > 1 else "n/a"
             stats.append(f"{statistics.mean(column):.5f} ± {spread}")
         print(f"{group:<23}" + " | ".join(stats))
+    if args.paired_deltas:
+        print("\nPaired deltas (percentage points; same column order; matching seeds only)")
+        for treatment, control in (("audio", "baseline"), ("visual", "baseline"),
+                                   ("av", "baseline"), ("av", "audio")):
+            left, right = by_seed.get(treatment, {}), by_seed.get(control, {})
+            seeds = sorted(left.keys() & right.keys())
+            if not seeds:
+                print(f"{treatment} - {control}: no matching seeds")
+                continue
+            differences = [[a - b for a, b in zip(left[seed], right[seed])] for seed in seeds]
+            stats = []
+            for column in zip(*differences):
+                spread = f"{statistics.stdev(column):.5f}" if len(column) > 1 else "n/a"
+                stats.append(f"{statistics.mean(column):+.5f} ± {spread}")
+            print(f"{treatment} - {control} (seeds={seeds}): " + " | ".join(stats))
 
 
 if __name__ == "__main__":