icassp2027-experiment/third_party/MuVAP/train_vap.py

import argparse
from pathlib import Path

import torch
from lightning.pytorch import Trainer, seed_everything
from lightning.pytorch.callbacks import (
    LearningRateMonitor,
    ModelCheckpoint,
    RichProgressBar,
)

from config.setup import load_config
from data.vap_dm import VAPDataModule
from models.release import load_weights, merge_config, resolve
from projection_window import ProjectionWindow
from tasks.vap_task import VAPTask


def parse_args():
    parser = argparse.ArgumentParser(description="Train or evaluate VAP")
    parser.add_argument(
        "--config", default=Path(__file__).resolve().parent / "config/yaml/vap_role.yaml"
    )
    parser.add_argument("--test", action="store_true")
    parser.add_argument("--checkpoint", help="Lightning .ckpt to resume or evaluate")
    parser.add_argument(
        "--weights",
        help="published release directory (or its weights.pt) to evaluate; "
        "the model architecture comes from the release, data paths from --config",
    )
    parser.add_argument("--seed", type=int, default=42)
    parser.add_argument("--wandb", action="store_true")
    parser.add_argument(
        "--wandb-project",
        default="VAP",
        help="W&B project the run is logged to (with --wandb)",
    )
    parser.add_argument("--name", help="run name used for the checkpoint directory")
    parser.add_argument(
        "--set",
        action="append",
        default=[],
        metavar="KEY=VALUE",
        help="override one config value, e.g. --set vap.temporal.pos_encoding=rope",
    )
    return parser.parse_args()


def apply_overrides(cfg, overrides):
    """Apply `--set a.b.c=value` onto a loaded config, parsed as YAML scalars."""
    import yaml

    for override in overrides:
        key, _, raw = override.partition("=")
        if not _:
            raise SystemExit(f"--set expects KEY=VALUE, got {override!r}")
        target = cfg
        *path, leaf = key.split(".")
        for step in path:
            if step not in target:
                raise SystemExit(f"unknown config key in --set {override!r}")
            target = target[step]
        if leaf not in target:
            raise SystemExit(f"unknown config key in --set {override!r}")
        target[leaf] = yaml.safe_load(raw)
    return cfg


def build_checkpoints(vap_cfg, checkpoint_dir):
    """Always keep `last.ckpt`; keep a selected `best.ckpt` only if asked to.

    Two callbacks, not one: a single `ModelCheckpoint` given both a monitor and
    `save_last=True` writes a `last.ckpt` that tracks the selected checkpoint
    rather than the final epoch, so the final epoch would never be kept.

    Selecting on a validation metric is only meaningful with a held-out split.
    Setting `checkpoint_monitor` to nothing says there is none, and the final
    epoch becomes the model the run produced.
    """
    last = ModelCheckpoint(dirpath=checkpoint_dir, save_top_k=0, save_last=True)
    monitor = vap_cfg.get("checkpoint_monitor", "val/loss")
    if not monitor:
        return [last]
    best = ModelCheckpoint(
        monitor=monitor,
        mode=vap_cfg.get("checkpoint_mode", "min"),
        save_top_k=1,
        dirpath=checkpoint_dir,
        filename="best",
    )
    return [best, last]


def build_trainer(cfg, logger, checkpoint_dir):
    """Trainer whose checkpoints and logs all land under `checkpoint_dir`."""
    vap_cfg = cfg["vap"]
    return Trainer(
        default_root_dir=checkpoint_dir,
        max_epochs=vap_cfg["max_epochs"],
        logger=logger,
        callbacks=[
            *build_checkpoints(vap_cfg, checkpoint_dir),
            RichProgressBar(),
            LearningRateMonitor("step"),
        ],
        log_every_n_steps=100,
        accelerator="auto",
        devices="auto",
        precision=vap_cfg.get("precision", "bf16-mixed"),
        gradient_clip_val=vap_cfg.get("gradient_clip_val", 1.0),
        accumulate_grad_batches=vap_cfg.get("accumulate_grad_batches", 1),
    )


def main():
    args = parse_args()
    if args.test and not (args.checkpoint or args.weights):
        raise SystemExit("--test needs --checkpoint (a .ckpt) or --weights (a release)")

    cfg = apply_overrides(load_config(args.config, "vap"), args.set)
    weights_path = None
    if args.weights:
        weights_path, release_config = resolve(args.weights)
        cfg = merge_config(cfg, release_config, "vap")
    seed_everything(args.seed, workers=True)
    torch.set_float32_matmul_precision(cfg["vap"].get("matmul_precision", "high"))

    logger = None
    run_id = args.name or "local"
    if args.wandb:
        from lightning.pytorch.loggers import WandbLogger

        logger = WandbLogger(project=args.wandb_project, name=args.name)
        run_id = args.name or logger.experiment.id

    projection = ProjectionWindow(**cfg["vap"]["projection_window"])
    print(f"VAP setup: arch={cfg['vap'].get('arch', 'mono')} {projection}")

    data = VAPDataModule(cfg, projection)
    model = VAPTask(cfg, projection)
    if weights_path is not None:
        load_weights(model.model, weights_path)
        print(f"loaded release weights from {weights_path}")
    trainer = build_trainer(cfg, logger, Path(cfg["checkpoint_dir"]) / run_id)

    if not args.test:
        trainer.fit(model, datamodule=data)
    data.setup("test")
    # A release is already loaded into the model; a ckpt_path would replace it.
    if weights_path is not None:
        ckpt_path = None
    else:
        # Without a selected best there is only the final epoch to fall back on.
        fallback = "best" if cfg["vap"].get("checkpoint_monitor", "val/loss") else "last"
        ckpt_path = args.checkpoint or fallback
    trainer.test(
        model,
        dataloaders=[data.tune_dataloader(), data.test_dataloader()],
        ckpt_path=ckpt_path,
    )


if __name__ == "__main__":
    main()