Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
+162
@@ -0,0 +1,162 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
import pandas as pd
|
||||
import os.path as osp
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
||||
topk_accuracy_multiple_timesteps,
|
||||
topk_recall_multiple_timesteps,
|
||||
tta)
|
||||
|
||||
|
||||
def eval_fusion(pred_scores_files, weights):
|
||||
### Comments: Per-frame scores cost too much space...
|
||||
# perframe_gt_targets_fusion = {}
|
||||
# perframe_pred_scores_fusion = {}
|
||||
# for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
# pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
# cfg = pred_scores['cfg']
|
||||
# assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
# perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
# perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# for session in perframe_pred_scores:
|
||||
# if session not in perframe_pred_scores_fusion:
|
||||
# perframe_pred_scores_fusion[session] = weight * perframe_pred_scores[session]
|
||||
# else:
|
||||
# perframe_pred_scores_fusion[session] += weight * perframe_pred_scores[session]
|
||||
# perframe_gt_targets_fusion[session] = perframe_gt_targets[session]
|
||||
|
||||
# # Compute results
|
||||
# result = compute_result['perframe'](
|
||||
# cfg,
|
||||
# np.concatenate(list(perframe_gt_targets_fusion.values()), axis=0),
|
||||
# np.concatenate(list(perframe_pred_scores_fusion.values()), axis=0),
|
||||
# )
|
||||
# logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
# cfg.DATA.METRICS, result['mean_AP']
|
||||
# ))
|
||||
|
||||
action_scores = None
|
||||
for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
if action_scores is None:
|
||||
action_scores = weight * pred_scores['action_scores']
|
||||
else:
|
||||
action_scores += weight * pred_scores['action_scores']
|
||||
# segment-level evaluation on EK55/EK100
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
predictions, labels, ids = [], [], []
|
||||
discarded_labels, discarded_ids = [], []
|
||||
for idx in segment_list['id']:
|
||||
segment = segment_list[segment_list['id'] == idx]
|
||||
start_step = int(np.floor(segment['start_f'].values.astype(int) / 30 * cfg.DATA.FPS))
|
||||
vid = segment['video'].values[0].strip()
|
||||
# preds = (perframe_pred_scores_fusion[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
# predictions.append(preds)
|
||||
labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
ids.append(idx)
|
||||
else:
|
||||
discarded_labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
discarded_ids.append(idx)
|
||||
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
# action_scores = np.concatenate(predictions)
|
||||
labels = np.concatenate(labels)
|
||||
ids = np.array(ids)
|
||||
|
||||
vi = get_marginal_indexes(actions, 'verb')
|
||||
ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
# action_probs = action_scores.reshape(-1, action_scores.shape[-1])
|
||||
# action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
||||
print(action_probs.shape)
|
||||
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
|
||||
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
if include_discarded:
|
||||
dlab = np.array(discarded_labels)
|
||||
dislab = np.array(discarded_ids)
|
||||
ids = np.concatenate([ids, dislab])
|
||||
num_disc = len(dlab)
|
||||
labels = np.concatenate([labels, dlab])
|
||||
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
for a in actions.iterrows()}
|
||||
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
k=5, classes=many_shot_verbs)
|
||||
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
k=5, classes=many_shot_nouns)
|
||||
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
k=5, classes=many_shot_actions)
|
||||
|
||||
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
verb_recalls, noun_recalls, action_recalls])
|
||||
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
indices = [
|
||||
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
('Verb', 'Mean Top-5 Recall'),
|
||||
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
('Noun', 'Mean Top-5 Recall'),
|
||||
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
('Action', 'Mean Top-5 Recall'),
|
||||
]
|
||||
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
1 / cfg.DATA.FPS, cfg.MODEL.LSTR.ANTICIPATION_LENGTH, dtype=str)
|
||||
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
index=pd.MultiIndex.from_tuples(indices))
|
||||
print(scores)
|
||||
|
||||
tta_verb = tta(verb_scores, verb_labels)
|
||||
tta_noun = tta(noun_scores, noun_labels)
|
||||
tta_action = tta(action_scores, action_labels)
|
||||
print(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_files', nargs='+', required=True)
|
||||
parser.add_argument('--weights', type=float, nargs='+', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert len(args.pred_scores_files) == len(args.weights)
|
||||
eval_fusion(args.pred_scores_files, args.weights)
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perframe(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perframe(args.pred_scores_file)
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perstage(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perstage'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
for stage_name in result:
|
||||
logging.info('Perframe m{} of stage {}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, stage_name, result[stage_name]['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perstage(args.pred_scores_file)
|
||||
@@ -0,0 +1,80 @@
|
||||
import argparse
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import os.path as osp
|
||||
|
||||
|
||||
def str2sec(string):
|
||||
hh, mm, ss = string.split(':')
|
||||
return float(hh) * 3600 + float(mm) * 60 + float(ss)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Generate targets')
|
||||
parser.add_argument('--dataset', type=str,
|
||||
choices=['EK55', 'EK100'], default='EK55')
|
||||
parser.add_argument('--data-path', type=str,
|
||||
default='/scratch/cluster/yzhao/Datasets/EPIC-KITCHENS/')
|
||||
parser.add_argument('--targets-path', type=str,
|
||||
default='/scratch/cluster/yzhao/Datasets/'
|
||||
'EPIC-KITCHENS/features/target_perframe/')
|
||||
parser.add_argument('--fps', type=int, default=30)
|
||||
parser.add_argument('--tick', type=float, default=7.5)
|
||||
parser.add_argument('--anno-path', type=str,
|
||||
default='external/rulstm/RULSTM/data/ek55/')
|
||||
parser.add_argument('--class-type', type=str, default='action',
|
||||
choices=['verb', 'noun', 'action'])
|
||||
args = parser.parse_args()
|
||||
|
||||
# initialize the targets
|
||||
targets = {}
|
||||
if args.class_type == 'action':
|
||||
num_classes = 2513 + 1
|
||||
elif args.class_type == 'verb':
|
||||
actions = pd.read_csv(osp.join(args.anno_path, 'actions.csv'))
|
||||
a_to_v = {a[1]['id'] + 1: a[1]['verb'] + 1
|
||||
for a in actions.iterrows()}
|
||||
num_classes = max(a_to_v.values()) + 1
|
||||
elif args.class_type == 'noun':
|
||||
actions = pd.read_csv(osp.join(args.anno_path, 'actions.csv'))
|
||||
a_to_n = {a[1]['id'] + 1: a[1]['noun'] + 1
|
||||
for a in actions.iterrows()}
|
||||
num_classes = max(a_to_n.values()) + 1
|
||||
|
||||
for subset in ['training', 'validation']:
|
||||
vids = [line.strip() for line in open('{}/{}_videos.csv'.format(args.anno_path, subset))]
|
||||
anno_per_video = {}
|
||||
for line in open('{}/{}.csv'.format(args.anno_path, subset)):
|
||||
idx, vid, f_start, f_end, c_verb, c_noun, c_verbnoun = line.strip().split(',')
|
||||
vid = vid.strip()
|
||||
if vid not in anno_per_video:
|
||||
anno_per_video[vid] = [(int(f_start), int(f_end), int(c_verb), int(c_noun), int(c_verbnoun))]
|
||||
else:
|
||||
anno_per_video[vid].append((int(f_start), int(f_end), int(c_verb), int(c_noun), int(c_verbnoun)))
|
||||
for vid in vids:
|
||||
feature = np.load('{}/features/rgb_kinetics_resnet50/{}.npy'.format(args.data_path, vid))
|
||||
feature_len = feature.shape[0]
|
||||
targets[vid] = np.zeros((feature_len,
|
||||
num_classes))
|
||||
targets[vid][:, 0] = 1
|
||||
for anno in anno_per_video[vid]:
|
||||
print(vid, anno)
|
||||
ss = int(np.floor(anno[0] / args.tick))
|
||||
ee = min(int(np.ceil(anno[1] / args.tick)), feature_len)
|
||||
if args.class_type == 'action':
|
||||
ci = anno[4]
|
||||
elif args.class_type == 'verb':
|
||||
ci = anno[2]
|
||||
elif args.class_type == 'noun':
|
||||
ci = anno[3]
|
||||
for si in range(ss, ee):
|
||||
targets[vid][si, ci + 1] = 1
|
||||
targets[vid][si, 0] = 0
|
||||
|
||||
print('Dumping npy files..')
|
||||
for vid in targets:
|
||||
np.save(args.targets_path + '{}.npy'.format(vid), targets[vid])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
|
||||
from rekognition_online_action_detection.engines import do_inference
|
||||
from rekognition_online_action_detection.models import build_model
|
||||
from rekognition_online_action_detection.utils.checkpointer import \
|
||||
setup_checkpointer
|
||||
from rekognition_online_action_detection.utils.env import setup_environment
|
||||
from rekognition_online_action_detection.utils.logger import setup_logger
|
||||
from rekognition_online_action_detection.utils.parser import load_cfg
|
||||
from utils import validate_cfg
|
||||
|
||||
|
||||
def main(cfg):
|
||||
validate_cfg(cfg)
|
||||
config_name = os.path.basename(cfg.OUTPUT_DIR)
|
||||
|
||||
print(f"cfg.MODEL.CHECKPOINT: {cfg.MODEL.CHECKPOINT}")
|
||||
print(f"cfg.OUTPUT_DIR: {cfg.OUTPUT_DIR}")
|
||||
|
||||
# Setup configurations
|
||||
device = setup_environment(cfg)
|
||||
checkpointer = setup_checkpointer(cfg, phase='test')
|
||||
logger = setup_logger(cfg, phase='test')
|
||||
|
||||
print(cfg)
|
||||
print(f"DATA.IGNORE_INDEX: {cfg.DATA.IGNORE_INDEX}")
|
||||
# Build model
|
||||
model = build_model(cfg, device)
|
||||
|
||||
# Load pretrained model
|
||||
checkpointer.load(model)
|
||||
|
||||
do_inference(
|
||||
cfg,
|
||||
model,
|
||||
device,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main(load_cfg())
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import wandb
|
||||
from rekognition_online_action_detection.criterions import build_criterion
|
||||
from rekognition_online_action_detection.datasets import build_data_loader
|
||||
from rekognition_online_action_detection.engines import do_train
|
||||
from rekognition_online_action_detection.models import build_model
|
||||
from rekognition_online_action_detection.optimizers import (build_optimizer,
|
||||
build_scheduler)
|
||||
from rekognition_online_action_detection.utils.checkpointer import \
|
||||
setup_checkpointer
|
||||
from rekognition_online_action_detection.utils.env import setup_environment
|
||||
from rekognition_online_action_detection.utils.logger import setup_logger
|
||||
from rekognition_online_action_detection.utils.parser import load_cfg
|
||||
from utils import parse_config
|
||||
|
||||
|
||||
def main(cfg):
|
||||
parse_config(cfg)
|
||||
# Setup configurations
|
||||
device = setup_environment(cfg)
|
||||
checkpointer = setup_checkpointer(cfg, phase="train")
|
||||
logger = setup_logger(cfg, phase="train")
|
||||
|
||||
print(cfg)
|
||||
print(f"cfg['SOLVER']['NUM_EPOCHS']: {cfg['SOLVER']['NUM_EPOCHS']}")
|
||||
print(f"cfg['MODEL']['CHECKPOINT']: {cfg['MODEL']['CHECKPOINT']}")
|
||||
|
||||
# Build data loaders
|
||||
data_loaders = {phase: build_data_loader(cfg, phase) for phase in cfg.SOLVER.PHASES}
|
||||
|
||||
# Build model
|
||||
model = build_model(cfg, device)
|
||||
|
||||
# Build criterion
|
||||
criterion = build_criterion(cfg, device)
|
||||
|
||||
# Build optimizer
|
||||
optimizer = build_optimizer(cfg, model)
|
||||
|
||||
# Load pretrained model and optimizer
|
||||
checkpointer.load(model, optimizer)
|
||||
|
||||
# Build scheduler
|
||||
scheduler = build_scheduler(cfg, optimizer, len(data_loaders["train"]))
|
||||
|
||||
do_train(
|
||||
cfg,
|
||||
data_loaders,
|
||||
model,
|
||||
criterion,
|
||||
optimizer,
|
||||
scheduler,
|
||||
device,
|
||||
checkpointer,
|
||||
logger,
|
||||
)
|
||||
|
||||
wandb.finish() if cfg.USE_WANDB else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(load_cfg())
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
import os
|
||||
|
||||
import wandb
|
||||
|
||||
|
||||
def parse_config(cfg):
|
||||
modality = get_modality(cfg)
|
||||
|
||||
cfg_filename = os.path.basename(cfg.OUTPUT_DIR)
|
||||
|
||||
os.environ["WANDB__SERVICE_WAIT"] = "3000"
|
||||
cfg_filename_abbr = cfg_filename.split('_1x_')[-1]
|
||||
wandb_postfix = generate_wandb_postfix(cfg, modality)
|
||||
scheduler_name = cfg.SOLVER.SCHEDULER.SCHEDULER_NAME
|
||||
learning_rate = cfg.SOLVER.BASE_LR
|
||||
weight_decay = cfg.SOLVER.WEIGHT_DECAY
|
||||
|
||||
cfg_filename = cfg_filename.replace('long_512', f'long_{cfg.MODEL.LSTR.LONG_MEMORY_SECONDS}')
|
||||
cfg_filename = cfg_filename.replace('work_8', f'work_{cfg.MODEL.LSTR.WORK_MEMORY_SECONDS}')
|
||||
|
||||
cfg.OUTPUT_DIR = os.path.join(os.path.dirname(cfg.OUTPUT_DIR), cfg_filename)
|
||||
|
||||
if cfg.SAVE_HPARAM:
|
||||
# save different dir containing hparam
|
||||
hparam_info = '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
cfg.OUTPUT_DIR = os.path.join(cfg.OUTPUT_DIR, hparam_info, modality)
|
||||
print(f"HPARAM: {hparam_info}")
|
||||
print(f"cfg.OUTPUT_DIR: {cfg.OUTPUT_DIR}")
|
||||
|
||||
wandb_name = cfg_filename_abbr + '_' + wandb_postfix
|
||||
|
||||
if cfg.SAVE_HPARAM:
|
||||
wandb_name += '_' + '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
|
||||
if cfg.USE_WANDB:
|
||||
print("*** init wandb ***")
|
||||
print(f"wandb_name: {wandb_name}")
|
||||
|
||||
def get_modality(cfg):
|
||||
modality = ''
|
||||
if 'motion' in cfg.INPUT.MODALITY:
|
||||
modality += 'A'
|
||||
if 'visual' in cfg.INPUT.MODALITY:
|
||||
modality += 'V'
|
||||
if 'object' in cfg.INPUT.MODALITY:
|
||||
modality += 'T'
|
||||
elif cfg['INPUT']['MODALITY'] == 'twostream':
|
||||
modality = 'AV'
|
||||
|
||||
if modality == '':
|
||||
raise ValueError(f"Unknown modality: {cfg['INPUT']['MODALITY']}")
|
||||
|
||||
return modality
|
||||
|
||||
|
||||
def generate_wandb_postfix(cfg, modality):
|
||||
visual_feature_type = '_'.join(cfg['INPUT']['VISUAL_FEATURE'].split('_')[1:])
|
||||
audio_feature_type = '_'.join(cfg['INPUT']['MOTION_FEATURE'].split('_')[1:])
|
||||
text_feature_type = '_'.join(cfg['INPUT']['OBJECT_FEATURE'].split('_')[1:])
|
||||
|
||||
if cfg.MODEL.LSTR.LONG_MEMORY_USE_PE == True:
|
||||
model_name = 'LSTR_TESTRA'
|
||||
else:
|
||||
model_name = 'TESTRA'
|
||||
|
||||
if '_aa' in cfg.OUTPUT_DIR:
|
||||
model_name += '_aa'
|
||||
|
||||
name_parts = [
|
||||
model_name,
|
||||
cfg['DATA']['DATA_NAME'],
|
||||
modality,
|
||||
# visual_feature_type,
|
||||
# audio_feature_type,
|
||||
# text_feature_type,
|
||||
# f"epoch_{cfg['SOLVER']['NUM_EPOCHS']}"
|
||||
]
|
||||
if cfg['MODEL']['CHECKPOINT']:
|
||||
name_parts.append(f"ckpt_{os.path.basename(cfg['MODEL']['CHECKPOINT']).split('.')[0]}")
|
||||
return '_'.join(name_parts)
|
||||
|
||||
def get_wandb_name(cfg, cfg_filename, modality):
|
||||
cfg_filename_abbr = cfg_filename.split('_1x_')[-1]
|
||||
wandb_postfix = generate_wandb_postfix(cfg, modality)
|
||||
scheduler_name = cfg.SOLVER.SCHEDULER.SCHEDULER_NAME
|
||||
learning_rate = cfg.SOLVER.BASE_LR
|
||||
weight_decay = cfg.SOLVER.WEIGHT_DECAY
|
||||
wandb_name = cfg_filename_abbr + '_' + wandb_postfix + '_' + '_'.join([f"lr_{learning_rate}", f"wd_{weight_decay}", f"schd_{scheduler_name}"])
|
||||
return wandb_name
|
||||
|
||||
def validate_cfg(cfg):
|
||||
if cfg.DATA.NUM_CLASSES != len(cfg.DATA.CLASS_NAMES):
|
||||
print(f"len(cfg.DATA.CLASS_NAMES): {len(cfg.DATA.CLASS_NAMES)} is not equal to cfg.DATA.NUM_CLASSES: {cfg.DATA.NUM_CLASSES}")
|
||||
cfg.DATA.CLASS_NAMES = cfg.DATA.CLASS_NAMES[:cfg.DATA.NUM_CLASSES]
|
||||
print(f"Adjusted cfg.DATA.CLASS_NAMES: {cfg.DATA.CLASS_NAMES}")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user