Add causal person fusion and MuVAP stage-3 baselines

third_party/EgoSpeak/MiniROAD/configs/EasyCom/mamba_aa_easycom.yaml

@@ -0,0 +1,32 @@
+model: 'MambaA'
+data_name: 'EasyCom_ANTICIPATION'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/EasyCom'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2'
+annotation_type: 'target_perframe'
+video_list_path: 'data_info/data_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 64 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 3
+embedding_dim: 2048
+hidden_dim: 1024
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100
+seed: 20

third_party/EgoSpeak/MiniROAD/configs/EasyCom/miniroad_aa_easycom.yaml

@@ -0,0 +1,32 @@
+model: 'MiniROADA'
+data_name: 'EasyCom_ANTICIPATION'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/EasyCom'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2'
+annotation_type: 'target_perframe'
+video_list_path: 'data_info/data_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 64 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 3 # [background, target speaker speaking, other person speaking]
+embedding_dim: 2048
+hidden_dim: 1024
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100
+seed: 20

third_party/EgoSpeak/MiniROAD/configs/YTConv/mamba_aa_ytconv.yaml

@@ -0,0 +1,32 @@
+model: 'MambaA'
+data_name: 'YTConv_ANTICIPATION'
+data_info_name: 'shard_0'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/YTConv/shard_0'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2' # or 'none'?
+annotation_type: 'target_perframe/case1'
+video_list_path: 'data_info/YTConv_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 32 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 2 # for YTConv pretraining, there is no 
+embedding_dim: 2048
+hidden_dim: 1024
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100

third_party/EgoSpeak/MiniROAD/configs/YTConv/miniroad_aa_ytconv.yaml

@@ -0,0 +1,32 @@
+model: 'MiniROADA'
+data_name: 'YTConv_ANTICIPATION'
+data_info_name: 'shard_0'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/YTConv/shard_0'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2'
+annotation_type: 'target_perframe'
+video_list_path: 'data_info/YTConv_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 64 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 2 # for pretraining, there is no target speaker
+embedding_dim: 2048
+hidden_dim: 1024
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100

third_party/EgoSpeak/MiniROAD/configs/ego4d/mamba_aa_ego4dshuffle.yaml

@@ -0,0 +1,36 @@
+model: 'MambaA'
+data_name: 'EGO4D_ANTICIPATION'
+data_info_name: 'EGO4DSHUFFLE'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/EGO4D'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2'
+annotation_type: 'target_perframe'
+video_list_path: 'data_info/data_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 64 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 3 # including background
+embedding_dim: 2048
+hidden_dim: 1024
+d_state: 16
+d_conv: 4
+expand: 2 
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100
+seed: 20

third_party/EgoSpeak/MiniROAD/configs/ego4d/miniroad_aa_ego4dshuffle.yaml

@@ -0,0 +1,32 @@
+model: 'MiniROADA'
+data_name: 'EGO4D_ANTICIPATION'
+data_info_name: 'EGO4DSHUFFLE'
+task: 'ANTICIPATION'
+loss: 'ANTICIPATION' # NONUNIFORM, originally
+metric: 'AP' # AP or cAP
+optimizer: 'AdamW'
+device: 'cuda'
+feature_pretrained: 'resnet50'
+root_path: './data/EGO4D'
+rgb_type: 'rgb_kinetics_resnet50'
+flow_type: 'audio_wav2vec2' # or 'none'?
+annotation_type: 'target_perframe'
+video_list_path: 'data_info/data_info.json'
+output_path: './output_cfg'
+window_size: 128
+batch_size: 64 
+test_batch_size: 1 
+# loss_offset: 4 # loss weight a_t when L - {loss_offset} <= t <= L
+num_epoch: 30
+lr: 0.0001
+weight_decay: 0.05
+num_workers: 4
+dropout: 0.20
+num_classes: 3
+embedding_dim: 2048
+hidden_dim: 1024
+num_layers: 1
+stride: 4
+anticipation_length: 10
+actionness: False
+ignore_index: -100

third_party/EgoSpeak/MiniROAD/configs/feature_sizes.py

@@ -0,0 +1,26 @@
+# dimension of features extracted from different models
+# used in feature_head, dataset, models
+
+FEATURE_SIZES = {
+    'rgb_anet_resnet50': 2048,
+    'flow_anet_resnet50': 2048,
+    'rgb_kinetics_bninception': 1024,
+    'flow_kinetics_bninception': 1024,
+    'rgb_farl': 512,
+    'rgb_kinetics_resnet50': 2048,
+    'rgb_kinetics_resnet50_debug': 2048,
+    'rgb_kinetics_resnet50_debug_resized': 2048,
+    'flow_kinetics_resnet50': 2048,
+    'flow_nv_kinetics_bninception': 1024,
+    'rgb_kinetics_i3d': 2048,
+    'flow_kinetics_i3d': 2048,
+    'audio_wav2vec2': 5120,
+    'audio_wav2vec2_left': 5120,
+    'audio_wav2vec2_right': 5120,
+    'text_word2vec': 300,
+    'text_fasttext_crawl': 300,
+    'text_gpt2_last_subword_window_size_0': 768,
+    'text_gpt2_last_subword_window_size_20': 768,
+    'text_gpt2_last_subword_window_size_50': 768,
+    'text_gpt2_last_subword_window_size_100': 768,
+}