Add causal person fusion and MuVAP stage-3 baselines

2026-09-10 06:54:22 +08:00
parent 0534c9d3d0
commit d426b4740a
225 changed files with 27716 additions and 119 deletions
@@ -0,0 +1,63 @@
+# Role-based VAP at 12.5 Hz, the rate Mimi natively codes at.
+#
+# `tap: latent` is Mimi's own frame rate, so this matches a full-duplex system
+# built on Mimi codes frame for frame: same frontend, same 80 ms decision grid.
+# Labels come from seg/vad12_5, written by 03_prep_vad_rate.py; the 25 Hz tree
+# is untouched. bin_sec stays at the paper's times, so the rate is the only
+# thing that changes against the 25 Hz runs - at 12.5 Hz they round to
+# 0.16 / 0.4 / 0.64 / 0.8, still a 2.0 s horizon.
+fisher_path: data/fisher
+checkpoint_dir: runs/role_mimi_12hz
+
+vap:
+  audio_encoder:
+    name: mimi
+    tap: latent            # 12.5 Hz, Mimi's own frame rate
+
+  arch: mono                 # mono | stereo
+  batch_size: 64
+  num_workers: 16
+  max_epochs: 5
+  lr: 0.001
+  weight_decay: 0.01
+  warmup_ratio: 0.1
+  min_lr: 0.0001
+
+  precision: bf16-mixed
+  gradient_clip_val: 1.0
+  accumulate_grad_batches: 1
+  matmul_precision: high
+  # Metric that selects best.ckpt. Leave empty when there is no held-out
+  # split to select on: last.ckpt is written either way.
+  checkpoint_monitor: val/loss
+  checkpoint_mode: min
+  # npy reads the segments written by preprocessing; raw seeks the same window
+  # in the source recording and resamples it.
+  source: npy
+  swap_channels: false         # stereo-only augmentation
+
+  projection_window:
+    # 1.4 s is 17.5 frames at 12.5 Hz, so the paper's times cannot be
+    # represented here. These keep the 4.0 s window exact - [18, 7, 7, 18] -
+    # where rounding [1.4, 0.6] gives 26 frames of history for a 25-frame window.
+    bin_sec: [1.44, 0.56, 0.56, 1.44]
+    num_hist_bins: 2
+    frame_hz: 12.5
+    threshold: 0.5
+    mode: role_relative
+    cut_center_frame: false
+
+  temporal:
+    d_model: 256
+    m_ff: 3
+    n_heads: 4
+    n_layers: 4
+    dropout: 0.1
+    activation: GELU
+    causal: true
+    pos_encoding: alibi        # alibi | rope | sinusoidal | none
+    ffn: mlp                   # mlp | swiglu
+    norm: layer                # layer | rms
+    # Size of the sinusoidal position table. Rotary and ALiBi extend to any
+    # length, so this bounds the sinusoidal encoding only.
+    max_frames: 4096