Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
+109
@@ -0,0 +1,109 @@
|
||||
checkpoint_dir: runs/asd
|
||||
|
||||
evaluation:
|
||||
# The official val_orig.csv, shipped inside the pack it scores.
|
||||
ava_groundtruth: data/AVA/packed/val/groundtruth.csv
|
||||
ava_predictions: runs/asd/ava_predictions.csv
|
||||
|
||||
# Each corpus keeps its pack beside its own media, so a dataset can be copied
|
||||
# to fast local storage on its own. AVA val is disjoint from everything and is
|
||||
# the only evaluation set. Measured overlap at the source-video level:
|
||||
#
|
||||
# AVA train / val 0 of 33 shared - a real held-out split
|
||||
# WASD train / val 161 of 161 shared - never held out, free to train on
|
||||
# MSDWild few.train / few.val 0 of 489 shared - a real held-out split
|
||||
# MSDWild few.train / many.val 0 of 172 shared - a real held-out split
|
||||
#
|
||||
# WASD val and both MSDWild val sets are trained on deliberately. WASD val is
|
||||
# not a held-out set to begin with. The MSDWild pair are: training on them
|
||||
# spends two genuine evaluation sets (23.4 h, and 16.9 h of the harder
|
||||
# many-speaker condition) to buy 40 h of training data. AVA val is untouched
|
||||
# and is what the reported mAP is computed on. Move any of them back under
|
||||
# `val` to recover them as evaluation sets.
|
||||
packed_asd:
|
||||
train:
|
||||
- data/AVA/packed/train
|
||||
- data/WASD/packed/train
|
||||
- data/WASD/packed/val
|
||||
- data/MSDWild/packed/few.train
|
||||
- data/MSDWild/packed/few.val
|
||||
- data/MSDWild/packed/many.val
|
||||
val:
|
||||
- data/AVA/packed/val
|
||||
|
||||
asd:
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
seed: 0
|
||||
|
||||
# Frozen audio frontend: cpc (256-dim) or mimi (512-dim, Kyutai). Both emit
|
||||
# 25 Hz features, which is the rate the visual frontend runs at.
|
||||
audio_encoder: cpc
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 1
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/mAP_official
|
||||
checkpoint_mode: max
|
||||
|
||||
# Batches are built to a frame budget rather than a fixed batch size, and
|
||||
# tracks longer than train_window are split into near-equal chunks so one
|
||||
# epoch still covers every frame. Each chunk after the first also reads the
|
||||
# projection window's history as unscored context, so a causal model never
|
||||
# has to predict from history it was not shown. See docs/preprocessing.md.
|
||||
train_window: 250
|
||||
frame_budget: 2048
|
||||
min_batch: 4
|
||||
max_batch: 16
|
||||
|
||||
# Validation batches whole tracks, grouped by length and right-padded. It is
|
||||
# forward-only, so it affords a much larger budget than training.
|
||||
val_frame_budget: 8192
|
||||
val_max_batch: 64
|
||||
|
||||
# Train-time augmentation. Audio mixing is the one that attacks false
|
||||
# positives: foreign speech is added while the label stays unchanged, so the
|
||||
# model has to reject speech it cannot see on the face. Donors are drawn from
|
||||
# a different clip, never from the batch.
|
||||
augment:
|
||||
audio_mix_prob: 0.5
|
||||
audio_snr_db: [-5.0, 5.0]
|
||||
visual: true
|
||||
crop_scale: [0.7, 1.0]
|
||||
rotate_degrees: 15.0
|
||||
# Ablation: replace the audio and relabel the sample silent. Off by default.
|
||||
substitute_audio_prob: 0.0
|
||||
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.00001
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.8, 0.6, 0.4, 0.2, 0.2, 0.4]
|
||||
num_hist_bins: 4
|
||||
frame_hz: 25
|
||||
threshold: 0.5
|
||||
mode: independent
|
||||
cut_center_frame: false
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 4
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
|
||||
temporal:
|
||||
d_model: 512
|
||||
m_ff: 4
|
||||
n_heads: 8
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
# ASD on a frozen Mimi frontend. Everything but the audio encoder matches
|
||||
# asd.yaml, so a difference against that run is attributable to the encoder.
|
||||
checkpoint_dir: runs/asd_mimi
|
||||
|
||||
evaluation:
|
||||
# The official val_orig.csv, shipped inside the pack it scores.
|
||||
ava_groundtruth: data/AVA/packed/val/groundtruth.csv
|
||||
ava_predictions: runs/asd_mimi/ava_predictions.csv
|
||||
|
||||
# Each corpus keeps its pack beside its own media, so a dataset can be copied
|
||||
# to fast local storage on its own. AVA val is disjoint from everything and is
|
||||
# the only evaluation set. Measured overlap at the source-video level:
|
||||
#
|
||||
# AVA train / val 0 of 33 shared - a real held-out split
|
||||
# WASD train / val 161 of 161 shared - never held out, free to train on
|
||||
# MSDWild few.train / few.val 0 of 489 shared - a real held-out split
|
||||
# MSDWild few.train / many.val 0 of 172 shared - a real held-out split
|
||||
#
|
||||
# WASD val and both MSDWild val sets are trained on deliberately. WASD val is
|
||||
# not a held-out set to begin with. The MSDWild pair are: training on them
|
||||
# spends two genuine evaluation sets (23.4 h, and 16.9 h of the harder
|
||||
# many-speaker condition) to buy 40 h of training data. AVA val is untouched
|
||||
# and is what the reported mAP is computed on. Move any of them back under
|
||||
# `val` to recover them as evaluation sets.
|
||||
packed_asd:
|
||||
train:
|
||||
- data/AVA/packed/train
|
||||
- data/WASD/packed/train
|
||||
- data/WASD/packed/val
|
||||
- data/MSDWild/packed/few.train
|
||||
- data/MSDWild/packed/few.val
|
||||
- data/MSDWild/packed/many.val
|
||||
val:
|
||||
- data/AVA/packed/val
|
||||
|
||||
asd:
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
seed: 0
|
||||
|
||||
# Frozen audio frontend: cpc (256-dim) or mimi (512-dim, Kyutai). Both emit
|
||||
# 25 Hz features, which is the rate the visual frontend runs at.
|
||||
audio_encoder:
|
||||
name: mimi
|
||||
tap: encoder # 25 Hz, matching the visual timeline
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 1
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/mAP_official
|
||||
checkpoint_mode: max
|
||||
|
||||
# Batches are built to a frame budget rather than a fixed batch size, and
|
||||
# tracks longer than train_window are split into near-equal chunks so one
|
||||
# epoch still covers every frame. Each chunk after the first also reads the
|
||||
# projection window's history as unscored context, so a causal model never
|
||||
# has to predict from history it was not shown. See docs/preprocessing.md.
|
||||
train_window: 250
|
||||
frame_budget: 2048
|
||||
min_batch: 4
|
||||
max_batch: 16
|
||||
|
||||
# Validation batches whole tracks, grouped by length and right-padded. It is
|
||||
# forward-only, so it affords a much larger budget than training.
|
||||
val_frame_budget: 8192
|
||||
val_max_batch: 64
|
||||
|
||||
# Train-time augmentation. Audio mixing is the one that attacks false
|
||||
# positives: foreign speech is added while the label stays unchanged, so the
|
||||
# model has to reject speech it cannot see on the face. Donors are drawn from
|
||||
# a different clip, never from the batch.
|
||||
augment:
|
||||
audio_mix_prob: 0.5
|
||||
audio_snr_db: [-5.0, 5.0]
|
||||
visual: true
|
||||
crop_scale: [0.7, 1.0]
|
||||
rotate_degrees: 15.0
|
||||
# Ablation: replace the audio and relabel the sample silent. Off by default.
|
||||
substitute_audio_prob: 0.0
|
||||
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.00001
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.8, 0.6, 0.4, 0.2, 0.2, 0.4]
|
||||
num_hist_bins: 4
|
||||
frame_hz: 25
|
||||
threshold: 0.5
|
||||
mode: independent
|
||||
cut_center_frame: false
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 4
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
|
||||
temporal:
|
||||
d_model: 512
|
||||
m_ff: 4
|
||||
n_heads: 8
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
# Role-based VAP: one mixed channel, 136 role-relative classes.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/vap_role
|
||||
|
||||
vap:
|
||||
arch: mono # mono | stereo
|
||||
batch_size: 64
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 1
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: npy
|
||||
swap_channels: false # stereo-only augmentation
|
||||
|
||||
projection_window:
|
||||
bin_sec: [1.4, 0.6, 0.6, 1.4]
|
||||
num_hist_bins: 2
|
||||
frame_hz: 25
|
||||
threshold: 0.5
|
||||
mode: role_relative
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 4
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp # mlp | swiglu
|
||||
norm: layer # layer | rms
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,58 @@
|
||||
# Role-based VAP at 50 Hz on CPC - the original VAP's own configuration.
|
||||
#
|
||||
# CPC's conv stack runs at 100 Hz and `downsample` halves it once per block, so
|
||||
# 50 Hz is one block instead of two. This is the rate and frontend of Ekstedt &
|
||||
# Skantze (2022); Mimi cannot reach it, its encoder tops out at 25 Hz.
|
||||
# Labels come from seg/vad50, written by 03_prep_vad_rate.py.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/role_cpc_50hz
|
||||
|
||||
vap:
|
||||
audio_encoder:
|
||||
name: cpc
|
||||
frame_hz: 50 # one downsample block instead of two
|
||||
|
||||
arch: mono # mono | stereo
|
||||
batch_size: 64
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 1
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: npy
|
||||
swap_channels: false # stereo-only augmentation
|
||||
|
||||
projection_window:
|
||||
bin_sec: [1.4, 0.6, 0.6, 1.4]
|
||||
num_hist_bins: 2
|
||||
frame_hz: 50
|
||||
threshold: 0.5
|
||||
mode: role_relative
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 4
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp # mlp | swiglu
|
||||
norm: layer # layer | rms
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,63 @@
|
||||
# Role-based VAP at 12.5 Hz, the rate Mimi natively codes at.
|
||||
#
|
||||
# `tap: latent` is Mimi's own frame rate, so this matches a full-duplex system
|
||||
# built on Mimi codes frame for frame: same frontend, same 80 ms decision grid.
|
||||
# Labels come from seg/vad12_5, written by 03_prep_vad_rate.py; the 25 Hz tree
|
||||
# is untouched. bin_sec stays at the paper's times, so the rate is the only
|
||||
# thing that changes against the 25 Hz runs - at 12.5 Hz they round to
|
||||
# 0.16 / 0.4 / 0.64 / 0.8, still a 2.0 s horizon.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/role_mimi_12hz
|
||||
|
||||
vap:
|
||||
audio_encoder:
|
||||
name: mimi
|
||||
tap: latent # 12.5 Hz, Mimi's own frame rate
|
||||
|
||||
arch: mono # mono | stereo
|
||||
batch_size: 64
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 1
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: npy
|
||||
swap_channels: false # stereo-only augmentation
|
||||
|
||||
projection_window:
|
||||
# 1.4 s is 17.5 frames at 12.5 Hz, so the paper's times cannot be
|
||||
# represented here. These keep the 4.0 s window exact - [18, 7, 7, 18] -
|
||||
# where rounding [1.4, 0.6] gives 26 frames of history for a 25-frame window.
|
||||
bin_sec: [1.44, 0.56, 0.56, 1.44]
|
||||
num_hist_bins: 2
|
||||
frame_hz: 12.5
|
||||
threshold: 0.5
|
||||
mode: role_relative
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 4
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp # mlp | swiglu
|
||||
norm: layer # layer | rms
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
# Speaker-based VAP (Ekstedt & Skantze, 2022): two audio channels, one speaker per
|
||||
# channel, and a 256-class codebook that projects one channel per row.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/vap_speaker
|
||||
|
||||
vap:
|
||||
arch: stereo
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 25
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,73 @@
|
||||
# Speaker-based VAP at 50 Hz on CPC - the original VAP's own configuration.
|
||||
#
|
||||
# CPC's conv stack runs at 100 Hz and `downsample` halves it once per block, so
|
||||
# 50 Hz is one block instead of two. This is the rate and frontend of Ekstedt &
|
||||
# Skantze (2022); Mimi cannot reach it, its encoder tops out at 25 Hz.
|
||||
# Labels come from seg/vad50, written by 03_prep_vad_rate.py.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/speaker_cpc_50hz
|
||||
|
||||
vap:
|
||||
audio_encoder:
|
||||
name: cpc
|
||||
frame_hz: 50 # one downsample block instead of two
|
||||
|
||||
arch: stereo
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 50
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,75 @@
|
||||
# Speaker-based VAP at 12.5 Hz, the rate Mimi natively codes at.
|
||||
#
|
||||
# `tap: latent` is Mimi's own frame rate, so this matches a full-duplex system
|
||||
# built on Mimi codes frame for frame: same frontend, same 80 ms decision grid.
|
||||
# Labels come from seg/vad12_5, written by 03_prep_vad_rate.py; the 25 Hz tree
|
||||
# is untouched. bin_sec stays at the paper's times, so the rate is the only
|
||||
# thing that changes against the 25 Hz runs - at 12.5 Hz they round to
|
||||
# 0.16 / 0.4 / 0.64 / 0.8, still a 2.0 s horizon.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/speaker_mimi_12hz
|
||||
|
||||
vap:
|
||||
audio_encoder:
|
||||
name: mimi
|
||||
tap: latent # 12.5 Hz, Mimi's own frame rate
|
||||
|
||||
arch: stereo
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 12.5
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,73 @@
|
||||
# Anchored mono VAP at 50 Hz on CPC - the original VAP's own configuration.
|
||||
#
|
||||
# CPC's conv stack runs at 100 Hz and `downsample` halves it once per block, so
|
||||
# 50 Hz is one block instead of two. This is the rate and frontend of Ekstedt &
|
||||
# Skantze (2022); Mimi cannot reach it, its encoder tops out at 25 Hz.
|
||||
# Labels come from seg/vad50, written by 03_prep_vad_rate.py.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/mono_cpc_50hz
|
||||
|
||||
vap:
|
||||
audio_encoder:
|
||||
name: cpc
|
||||
frame_hz: 50 # one downsample block instead of two
|
||||
|
||||
arch: anchored # one mixed channel, anchored by past VAD
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 50
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,79 @@
|
||||
# Speaker-based VAP from ONE channel, anchored by voice activity.
|
||||
#
|
||||
# The 256-class codebook names channels, which mono audio cannot identify. Here
|
||||
# the identity comes from the VAD instead: the model hears the mixture and is
|
||||
# told who held the floor strictly before the frame it is labelling, so the two
|
||||
# rows stay attached to real speakers without a second channel. The anchor is
|
||||
# delayed by one frame because speaker_based bins the future from `t` inclusive.
|
||||
#
|
||||
# Frozen Mimi frontend, so this is comparable to vap-speaker-mimi on everything
|
||||
# but the number of channels.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/vap_speaker_mono_mimi
|
||||
|
||||
vap:
|
||||
# Frozen 37.8M Mimi encoder. Its 512-dim features are bridged onto d_model by
|
||||
# the model's encoder projection.
|
||||
audio_encoder:
|
||||
name: mimi
|
||||
tap: encoder # 25 Hz, matching the label grid
|
||||
|
||||
arch: anchored # one mixed channel, anchored by past VAD
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 25
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
@@ -0,0 +1,77 @@
|
||||
# Anchored mono VAP at 12.5 Hz, the rate Mimi natively codes at.
|
||||
#
|
||||
# `tap: latent` is Mimi's own frame rate, so this matches a full-duplex system
|
||||
# built on Mimi codes frame for frame: same frontend, same 80 ms decision grid.
|
||||
# Labels come from seg/vad12_5, written by 03_prep_vad_rate.py; the 25 Hz tree
|
||||
# is untouched. bin_sec stays at the paper's times, so the rate is the only
|
||||
# thing that changes against the 25 Hz runs - at 12.5 Hz they round to
|
||||
# 0.16 / 0.4 / 0.64 / 0.8, still a 2.0 s horizon.
|
||||
fisher_path: data/fisher
|
||||
checkpoint_dir: runs/mono_mimi_12hz
|
||||
|
||||
vap:
|
||||
# Frozen 37.8M Mimi encoder. Its 512-dim features are bridged onto d_model by
|
||||
# the model's encoder projection.
|
||||
audio_encoder:
|
||||
name: mimi
|
||||
tap: latent # 12.5 Hz, Mimi's own frame rate
|
||||
|
||||
arch: anchored # one mixed channel, anchored by past VAD
|
||||
batch_size: 32
|
||||
num_workers: 16
|
||||
max_epochs: 5
|
||||
lr: 0.001
|
||||
weight_decay: 0.01
|
||||
warmup_ratio: 0.1
|
||||
min_lr: 0.0001
|
||||
|
||||
precision: bf16-mixed
|
||||
gradient_clip_val: 1.0
|
||||
accumulate_grad_batches: 2
|
||||
matmul_precision: high
|
||||
# Metric that selects best.ckpt. Leave empty when there is no held-out
|
||||
# split to select on: last.ckpt is written either way.
|
||||
checkpoint_monitor: val/loss
|
||||
checkpoint_mode: min
|
||||
# npy reads the segments written by preprocessing; raw seeks the same window
|
||||
# in the source recording and resamples it.
|
||||
source: raw
|
||||
swap_channels: true # channel order is arbitrary; swap both audio and VAD
|
||||
|
||||
projection_window:
|
||||
bin_sec: [0.2, 0.4, 0.6, 0.8]
|
||||
num_hist_bins: 0 # every bin is a future bin, as in the original repo
|
||||
frame_hz: 12.5
|
||||
threshold: 0.5
|
||||
mode: speaker_based
|
||||
cut_center_frame: false
|
||||
|
||||
temporal:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 1
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
|
||||
cross:
|
||||
d_model: 256
|
||||
m_ff: 3
|
||||
n_heads: 4
|
||||
n_layers: 3
|
||||
dropout: 0.1
|
||||
activation: GELU
|
||||
causal: true
|
||||
pos_encoding: alibi # alibi | rope | sinusoidal | none
|
||||
ffn: mlp
|
||||
norm: layer
|
||||
# Size of the sinusoidal position table. Rotary and ALiBi extend to any
|
||||
# length, so this bounds the sinusoidal encoding only.
|
||||
max_frames: 4096
|
||||
Reference in New Issue
Block a user