icassp2027-experiment/egospeak_omni_finetune.py

"""Stage-two selective MiniROADA adaptation from a trained Omni residual.

The released input features and Omni states remain frozen. This module only
enables gradients in selected MiniROADA temporal/forecast modules and the
existing residual. Stage-one checkpoints are loaded strictly, without resetting
the learned residual or requiring the original checkpoint to be available.
"""

from __future__ import annotations

from copy import deepcopy
from pathlib import Path
from typing import Any, Literal

import torch

from easycom_head import NUM_HORIZONS, OFFICIAL_TRAIN_SESSIONS
from egospeak_omni_residual import (
    EgoSpeakOmniResidual,
    MiniROADABase,
    load_torch_payload,
    normalize_session_name,
)

BaseTrainable = Literal["none", "heads", "gru-heads"]
STAGE_ONE_FORMAT = "egospeak-omni-zero-residual-v1"
STAGE_TWO_FORMAT = "egospeak-omni-selective-finetune-v1"


def validate_training_sessions(values: list[str]) -> None:
    normalized = [normalize_session_name(value) for value in values]
    if not normalized or len(normalized) != len(set(normalized)):
        raise ValueError("Training sessions must be nonempty and unique")
    if not set(normalized).issubset(OFFICIAL_TRAIN_SESSIONS):
        raise ValueError("Training may only use EasyCom Sessions 4--12")


def load_stage_one_checkpoint(
    path: Path,
) -> tuple[EgoSpeakOmniResidual, dict[str, Any]]:
    path = path.expanduser().resolve()
    if not path.is_file():
        raise FileNotFoundError(path)
    payload = load_torch_payload(path)
    if not isinstance(payload, dict) or payload.get("format") != STAGE_ONE_FORMAT:
        raise ValueError(
            "--stage-one-checkpoint must be a stage-one residual checkpoint"
        )
    config = payload.get("model_config")
    if not isinstance(config, dict):
        raise TypeError("Stage-one checkpoint lacks model_config")
    required = {
        "omni_dim",
        "fusion_dim",
        "tail_embedding_dim",
        "num_tail_bins",
        "dropout",
    }
    if set(config) != required or config["num_tail_bins"] != NUM_HORIZONS:
        raise ValueError("Incompatible stage-one residual model_config")
    training = payload.get("training_config")
    if not isinstance(training, dict) or int(training.get("epochs", 0)) <= 0:
        raise ValueError("Stage-one checkpoint lacks a positive training epoch budget")
    validate_training_sessions(payload.get("train_sessions", []))
    base_config = payload.get("base_model_config", {})
    if not isinstance(base_config, dict):
        raise TypeError("Invalid base_model_config")
    # Released v1 checkpoints omit base_model_config and use default MiniROADA.
    model = EgoSpeakOmniResidual(MiniROADABase(**base_config), **config)
    model.load_state_dict(payload["model_state_dict"], strict=True)
    if any(not torch.isfinite(value).all() for value in model.state_dict().values()):
        raise ValueError("Stage-one checkpoint contains nonfinite parameters")
    model.requires_grad_(False)
    model.eval()
    return model, payload


class EgoSpeakOmniFinetune(EgoSpeakOmniResidual):
    """Continuation with explicit base policy and intact stage-one logits."""

    def __init__(
        self, *args: Any, base_trainable: BaseTrainable = "gru-heads", **kwargs: Any
    ):
        super().__init__(*args, **kwargs)
        if base_trainable not in ("none", "heads", "gru-heads"):
            raise ValueError(f"Invalid base-trainable policy: {base_trainable}")
        self.base_trainable = base_trainable
        self.requires_grad_(True)
        self.base.requires_grad_(False)
        if base_trainable in ("heads", "gru-heads"):
            self.base.anticipation_layer.requires_grad_(True)
            self.base.f_classification.requires_grad_(True)
        if base_trainable == "gru-heads":
            self.base.gru.requires_grad_(True)
        self.train(False)

    @classmethod
    def from_stage_one(
        cls,
        reference: EgoSpeakOmniResidual,
        config: dict[str, Any],
        *,
        base_trainable: BaseTrainable = "gru-heads",
    ) -> EgoSpeakOmniFinetune:
        model = cls(deepcopy(reference.base), **config, base_trainable=base_trainable)
        # Copy rather than share weights with the immutable first-stage reference.
        model.load_state_dict(reference.state_dict(), strict=True)
        return model

    def train(self, mode: bool = True) -> EgoSpeakOmniFinetune:
        super().train(mode)
        # Keep frozen layer1's dropout off. Enabling GRU/head train mode does
        # not turn the original frozen RGB+audio projection back on.
        if self.base_trainable in ("heads", "gru-heads"):
            self.base.anticipation_layer.train(mode)
            self.base.f_classification.train(mode)
        if self.base_trainable == "gru-heads":
            self.base.gru.train(mode)
        return self

    def optimizer_groups(
        self, *, fusion_lr: float, base_lr: float
    ) -> list[dict[str, Any]]:
        if fusion_lr <= 0 or base_lr <= 0:
            raise ValueError("Both learning rates must be positive")
        fusion = [
            p
            for name, p in self.named_parameters()
            if p.requires_grad and not name.startswith("base.")
        ]
        base = [
            p
            for name, p in self.named_parameters()
            if p.requires_grad and name.startswith("base.")
        ]
        groups = [{"name": "fusion", "params": fusion, "lr": fusion_lr}]
        if base:
            groups.append({"name": "miniroada", "params": base, "lr": base_lr})
        return groups

    def forward_last(
        self,
        rgb: torch.Tensor,
        audio: torch.Tensor,
        omni: torch.Tensor,
        tail_bin: torch.Tensor,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        hidden, _ = self.base.encode(rgb, audio)
        ego_hidden = hidden[:, -1, :]
        base_logits = self.base.anticipation_logits(ego_hidden)
        # No no_grad/detach: gradients must reach the selected MiniROADA GRU.
        residual = self._residual_logits(ego_hidden, omni, tail_bin)
        return base_logits + residual, base_logits

    def forward_sequence_chunk(
        self,
        rgb: torch.Tensor,
        audio: torch.Tensor,
        omni: torch.Tensor,
        tail_bin: torch.Tensor,
        recurrent_state: torch.Tensor | None,
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        hidden, next_state = self.base.encode(rgb, audio, recurrent_state)
        base_logits = self.base.anticipation_logits(hidden)
        residual = self._residual_logits(hidden, omni, tail_bin)
        return base_logits + residual, base_logits, next_state


def base_model_config(base: MiniROADABase) -> dict[str, Any]:
    return {
        "rgb_dim": base.rgb_dim,
        "audio_dim": base.audio_dim,
        "embedding_dim": base.embedding_dim,
        "hidden_dim": base.hidden_dim,
        "num_layers": base.num_layers,
        "num_horizons": base.num_horizons,
        "num_classes": base.num_classes,
        "dropout": base.layer1[-1].p,
    }


def load_stage_two_checkpoint(
    path: Path,
) -> tuple[EgoSpeakOmniFinetune, dict[str, Any]]:
    payload = load_torch_payload(path)
    if not isinstance(payload, dict) or payload.get("format") != STAGE_TWO_FORMAT:
        raise ValueError("Not a stage-two fine-tuning checkpoint")
    model = EgoSpeakOmniFinetune(
        MiniROADABase(**payload["base_model_config"]),
        **payload["model_config"],
        base_trainable=payload["base_trainable"],
    )
    model.load_state_dict(payload["model_state_dict"], strict=True)
    model.eval()
    return model, payload