icassp2027-experiment/summarize_local_baselines.py

"""Print B0/B1 activity results in percent, with one explicit AP protocol."""

import argparse
import json
from pathlib import Path
import re
import statistics


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--root", type=Path, default=Path("results/easycom_local_b1"))
    parser.add_argument("--protocol", choices=("causal_valid_only", "egospeak_full_timeline"), default="causal_valid_only")
    parser.add_argument("--dataset", choices=("easycom", "ego4d"), help="Reject results from a different dataset")
    parser.add_argument("--paired-deltas", action="store_true", help="Compare matching seeds to baseline, plus AV minus audio")
    args = parser.parse_args()
    files = sorted(args.root.glob("*/metrics.json"))
    if not files:
        raise SystemExit(f"No */metrics.json in {args.root}")
    print(f"Protocol: {args.protocol}; all AP/mAP numbers are percentages")
    print(f"{'run':<23} {'0.2s mAP':>11} {'BG AP':>11} {'wearer AP':>11} {'others AP':>11} {'mean mAP':>11}")
    groups, by_seed = {}, {}
    comparison_signature = None
    for path in files:
        metrics = json.loads(path.read_text())
        if args.dataset is not None and metrics.get("dataset") != args.dataset:
            raise ValueError(f"Dataset mismatch in {path}: {metrics.get('dataset')}")
        if args.paired_deltas:
            signature = (metrics.get("dataset"), metrics.get("sessions"), metrics.get("checkpoint_epoch"),
                         metrics.get("checkpoint_train_sessions"), metrics.get("checkpoint_dev_sessions"),
                         metrics.get("loss"), metrics.get("layer_fusion"))
            if comparison_signature is not None and signature != comparison_signature:
                raise ValueError(f"Paired runs have different splits/epochs/loss/fusion: {path}")
            comparison_signature = signature
        score = metrics[args.protocol]
        first = score["horizons"][0]
        ap = first["class_ap"]
        values = [100 * value for value in (first["map"], ap["background"],
                  ap["camera_wearer_speaking"], ap["other_people_speaking"], score["mean_map"])]
        name = path.parent.name
        print(f"{name:<23}" + "".join(f" {v:11.5f}" for v in values))
        group = re.sub(r"_seed\d+$", "", name)
        groups.setdefault(group, []).append(values)
        match = re.fullmatch(r"(.+)_seed(\d+)", name)
        if match:
            by_seed.setdefault(match[1], {})[int(match[2])] = values
    print("\nMean ± sample standard deviation (AP/mAP percentage points)")
    for group, rows in sorted(groups.items()):
        stats = []
        for column in zip(*rows):
            spread = f"{statistics.stdev(column):.5f}" if len(column) > 1 else "n/a"
            stats.append(f"{statistics.mean(column):.5f} ± {spread}")
        print(f"{group:<23}" + " | ".join(stats))
    if args.paired_deltas:
        print("\nPaired deltas (percentage points; same column order; matching seeds only)")
        for treatment, control in (("audio", "baseline"), ("visual", "baseline"),
                                   ("av", "baseline"), ("av", "audio")):
            left, right = by_seed.get(treatment, {}), by_seed.get(control, {})
            seeds = sorted(left.keys() & right.keys())
            if not seeds:
                print(f"{treatment} - {control}: no matching seeds")
                continue
            differences = [[a - b for a, b in zip(left[seed], right[seed])] for seed in seeds]
            stats = []
            for column in zip(*differences):
                spread = f"{statistics.stdev(column):.5f}" if len(column) > 1 else "n/a"
                stats.append(f"{statistics.mean(column):+.5f} ± {spread}")
            print(f"{treatment} - {control} (seeds={seeds}): " + " | ".join(stats))


if __name__ == "__main__":
    main()