wx44wx/icassp2027-experiment
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add causal person fusion and MuVAP stage-3 baselines

Browse Source
This commit is contained in:
Xin Wang
2026-09-10 06:54:22 +08:00
parent 0534c9d3d0
commit d426b4740a
225 changed files with 27716 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom.yaml
Vendored
+44
View File
@@ -0,0 +1,44 @@
DATA:
DATA_NAME: 'EasyCom'
INPUT:
MODALITY: 'visual+motion' # visual+motion+object
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
MOTION_FEATURE: 'audio_wav2vec2' # audio
OBJECT_FEATURE: 'audio_wav2vec2'
TARGET_PERFRAME: 'target_perframe' # 3 classes
MODEL:
MODEL_NAME: 'LSTR'
FEATURE_HEAD:
LINEAR_ENABLED: True
LINEAR_OUT_FEATURES: 1024
LSTR:
NUM_HEADS: 16
DIM_FEEDFORWARD: 1024
DROPOUT: 0.2
ACTIVATION: 'relu'
LONG_MEMORY_SECONDS: 512
LONG_MEMORY_SAMPLE_RATE: 4
LONG_MEMORY_USE_PE: True
WORK_MEMORY_SECONDS: 8
WORK_MEMORY_SAMPLE_RATE: 1
ANTICIPATION_SECONDS: 2
ANTICIPATION_SAMPLE_RATE: 1
ENC_MODULE: [
[16, 1, True], [32, 2, True]
]
DEC_MODULE: [-1, 2, True]
CRITERIONS: [['MCE', {}]]
DATA_LOADER:
BATCH_SIZE: 16
NUM_WORKERS: 8
PIN_MEMORY: True
SOLVER:
NUM_EPOCHS: 50
OPTIMIZER: 'adam'
BASE_LR: 7e-05
WEIGHT_DECAY: 5e-05
SCHEDULER:
SCHEDULER_NAME: 'warmup_cosine'
WARMUP_FACTOR: 0.3
WARMUP_EPOCHS: 10.0
WARMUP_METHOD: 'linear'
third_party/EgoSpeak/LSTR/configs/EasyCom/LSTR/lstr_aa_easycom_from_ytturn.yaml
Vendored
+45
View File
@@ -0,0 +1,45 @@
DATA:
DATA_NAME: 'EasyCom'
INPUT:
MODALITY: 'visual+motion' # visual+motion+object
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
MOTION_FEATURE: 'audio_wav2vec2' # audio
OBJECT_FEATURE: 'audio_wav2vec2'
TARGET_PERFRAME: 'target_perframe' # 3 classes
MODEL:
CHECKPOINT: 'checkpoints/lstr_ytconv_pretrained_AV.pth'
MODEL_NAME: 'LSTR'
FEATURE_HEAD:
LINEAR_ENABLED: True
LINEAR_OUT_FEATURES: 1024
LSTR:
NUM_HEADS: 16
DIM_FEEDFORWARD: 1024
DROPOUT: 0.2
ACTIVATION: 'relu'
LONG_MEMORY_SECONDS: 512
LONG_MEMORY_SAMPLE_RATE: 4
LONG_MEMORY_USE_PE: True
WORK_MEMORY_SECONDS: 8
WORK_MEMORY_SAMPLE_RATE: 1
ANTICIPATION_SECONDS: 2
ANTICIPATION_SAMPLE_RATE: 1
ENC_MODULE: [
[16, 1, True], [32, 2, True]
]
DEC_MODULE: [-1, 2, True]
CRITERIONS: [['MCE', {}]]
DATA_LOADER:
BATCH_SIZE: 16
NUM_WORKERS: 8
PIN_MEMORY: True
SOLVER:
NUM_EPOCHS: 50
OPTIMIZER: 'adam'
BASE_LR: 7e-05
WEIGHT_DECAY: 5e-05
SCHEDULER:
SCHEDULER_NAME: 'warmup_cosine'
WARMUP_FACTOR: 0.3
WARMUP_EPOCHS: 10.0
WARMUP_METHOD: 'linear'
third_party/EgoSpeak/LSTR/configs/YTConv/LSTR/lstr_aa_ytconv.yaml
Vendored
+46
View File
@@ -0,0 +1,46 @@
DATA:
DATA_NAME: 'YTConv'
DATA_INFO: 'data/YTConv_info.json'
INPUT:
MODALITY: 'visual+motion' # visual+motion+object
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
MOTION_FEATURE: 'audio_wav2vec2' # audio
OBJECT_FEATURE: 'audio_wav2vec2'
TARGET_PERFRAME: 'target_perframe' # 3 classes
MODEL:
MODEL_NAME: 'LSTR'
FEATURE_HEAD:
LINEAR_ENABLED: True
LINEAR_OUT_FEATURES: 1024
LSTR:
NUM_HEADS: 16
DIM_FEEDFORWARD: 1024
DROPOUT: 0.2
ACTIVATION: 'relu'
LONG_MEMORY_SECONDS: 512
LONG_MEMORY_SAMPLE_RATE: 4
LONG_MEMORY_USE_PE: True
WORK_MEMORY_SECONDS: 8
WORK_MEMORY_SAMPLE_RATE: 1
ANTICIPATION_SECONDS: 2
ANTICIPATION_SAMPLE_RATE: 1
ENC_MODULE: [
[16, 1, True], [32, 2, True]
]
DEC_MODULE: [-1, 2, True]
CRITERIONS: [['MCE', {}]]
DATA_LOADER:
BATCH_SIZE: 16
NUM_WORKERS: 8
PIN_MEMORY: True
SOLVER:
SAVE_EVERY_EPOCHS: 5
NUM_EPOCHS: 50
OPTIMIZER: 'adam'
BASE_LR: 7e-05
WEIGHT_DECAY: 5e-05
SCHEDULER:
SCHEDULER_NAME: 'warmup_cosine'
WARMUP_FACTOR: 0.3
WARMUP_EPOCHS: 10.0
WARMUP_METHOD: 'linear'
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle.yaml
Vendored
+45
View File
@@ -0,0 +1,45 @@
DATA:
DATA_NAME: 'EGO4DSHUFFLE'
# IGNORE_INDEX: 0 # calculate mAP w/o target not speaking (background)
INPUT:
MODALITY: 'visual+motion' # audio+visual
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
MOTION_FEATURE: 'audio_wav2vec2' # audio
OBJECT_FEATURE: 'rgb_kinetics_resnet50'
TARGET_PERFRAME: 'target_perframe' # 3 classes
MODEL:
MODEL_NAME: 'LSTR'
FEATURE_HEAD:
LINEAR_ENABLED: True
LINEAR_OUT_FEATURES: 1024
LSTR:
NUM_HEADS: 16
DIM_FEEDFORWARD: 1024
DROPOUT: 0.2
ACTIVATION: 'relu'
LONG_MEMORY_SECONDS: 512
LONG_MEMORY_SAMPLE_RATE: 4
LONG_MEMORY_USE_PE: True
WORK_MEMORY_SECONDS: 8
WORK_MEMORY_SAMPLE_RATE: 1
ANTICIPATION_SECONDS: 2
ANTICIPATION_SAMPLE_RATE: 1
ENC_MODULE: [
[16, 1, True], [32, 2, True]
]
DEC_MODULE: [-1, 2, True]
CRITERIONS: [['MCE', {}]]
DATA_LOADER:
BATCH_SIZE: 16
NUM_WORKERS: 8
PIN_MEMORY: True
SOLVER:
NUM_EPOCHS: 50
OPTIMIZER: 'adam'
BASE_LR: 7e-05
WEIGHT_DECAY: 5e-05
SCHEDULER:
SCHEDULER_NAME: 'warmup_cosine'
WARMUP_FACTOR: 0.3
WARMUP_EPOCHS: 10.0
WARMUP_METHOD: 'linear'
third_party/EgoSpeak/LSTR/configs/ego4d/LSTR/lstr_aa_ego4dshuffle_from_ytconv.yaml
Vendored
+46
View File
@@ -0,0 +1,46 @@
DATA:
DATA_NAME: 'EGO4DSHUFFLE'
# IGNORE_INDEX: 0 # calculate mAP w/o target not speaking (background)
INPUT:
MODALITY: 'visual+motion' # audio+visual
VISUAL_FEATURE: 'rgb_kinetics_resnet50'
MOTION_FEATURE: 'audio_wav2vec2' # audio
OBJECT_FEATURE: 'rgb_kinetics_resnet50'
TARGET_PERFRAME: 'target_perframe' # 3 classes
MODEL:
CHECKPOINT: 'checkpoints/lstr_ytconv_pretrained_AV.pth'
MODEL_NAME: 'LSTR'
FEATURE_HEAD:
LINEAR_ENABLED: True
LINEAR_OUT_FEATURES: 1024
LSTR:
NUM_HEADS: 16
DIM_FEEDFORWARD: 1024
DROPOUT: 0.2
ACTIVATION: 'relu'
LONG_MEMORY_SECONDS: 512
LONG_MEMORY_SAMPLE_RATE: 4
LONG_MEMORY_USE_PE: True
WORK_MEMORY_SECONDS: 8
WORK_MEMORY_SAMPLE_RATE: 1
ANTICIPATION_SECONDS: 2
ANTICIPATION_SAMPLE_RATE: 1
ENC_MODULE: [
[16, 1, True], [32, 2, True]
]
DEC_MODULE: [-1, 2, True]
CRITERIONS: [['MCE', {}]]
DATA_LOADER:
BATCH_SIZE: 16
NUM_WORKERS: 8
PIN_MEMORY: True
SOLVER:
NUM_EPOCHS: 50
OPTIMIZER: 'adam'
BASE_LR: 7e-05
WEIGHT_DECAY: 5e-05
SCHEDULER:
SCHEDULER_NAME: 'warmup_cosine'
WARMUP_FACTOR: 0.3
WARMUP_EPOCHS: 10.0
WARMUP_METHOD: 'linear'