Add higher-capacity last-four-layer fusion

2026-08-26 13:06:18 +08:00
parent 43e0e14cf5
commit f45ea9cbc6
9 changed files with 397 additions and 17 deletions
@@ -328,6 +328,7 @@ def main() -> None:
            "checkpoint_epoch": checkpoint_payload.get("epoch"),
            "checkpoint_train_sessions": checkpoint_payload.get("train_sessions"),
            "checkpoint_dev_sessions": checkpoint_payload.get("dev_sessions"),
+            "layer_fusion": head.layer_fusion,
            "layer_mixture_weights_oldest_to_newest": (
                head.layer_mixture_weights().detach().float().cpu().tolist()
                if head.layer_mixture_weights() is not None