Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
+4
@@ -0,0 +1,4 @@
|
||||
from .train_builder import build_trainer
|
||||
from .train import train_one_epoch
|
||||
from .eval_builder import build_eval
|
||||
from .eval import Evaluate
|
||||
+288
@@ -0,0 +1,288 @@
|
||||
import json
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import wandb
|
||||
from tqdm import tqdm
|
||||
from trainer.eval_builder import EVAL
|
||||
from utils import *
|
||||
from utils import (event_accuracy, perframe_average_precision,
|
||||
thumos_postprocessing)
|
||||
|
||||
|
||||
@EVAL.register("OAD")
|
||||
class Evaluate(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(Evaluate, self).__init__()
|
||||
self.data_processing = thumos_postprocessing if 'THUMOS' in cfg['data_name'] else None
|
||||
self.metric = cfg['metric']
|
||||
self.eval_method = perframe_average_precision
|
||||
self.event_eval_method = event_accuracy
|
||||
self.all_class_names = json.load(open(cfg['video_list_path']))[cfg["data_name"].split('_')[0]]['class_index']
|
||||
self.cfg = cfg
|
||||
self.ignore_index = cfg['ignore_index'] if 'ignore_index' in cfg else -100
|
||||
|
||||
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path):
|
||||
device = "cuda:0"
|
||||
model.eval()
|
||||
eval_preds = {}
|
||||
gt_targets_dict = {}
|
||||
|
||||
with torch.no_grad():
|
||||
pred_scores, gt_targets = [], []
|
||||
start = time.time()
|
||||
eval_loss = 0.0
|
||||
for it, (vid_name, rgb_input, flow_input, target) in enumerate(tqdm(dataloader, desc='Evaluation:', leave=False)):
|
||||
rgb_input, flow_input, target = rgb_input.to(device), flow_input.to(device), target.to(device)
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
|
||||
if criterion != None:
|
||||
loss = criterion(out_dict, target)
|
||||
eval_loss += loss.item()
|
||||
|
||||
pred_logit = out_dict['logits']
|
||||
prob_val = pred_logit.squeeze().cpu().numpy()
|
||||
target_batch = target.squeeze().cpu().numpy()
|
||||
pred_scores += list(prob_val)
|
||||
gt_targets += list(target_batch)
|
||||
|
||||
eval_preds[vid_name] = prob_val
|
||||
|
||||
end = time.time()
|
||||
num_frames = len(gt_targets)
|
||||
|
||||
# save prediction scores
|
||||
# if result_path != None:
|
||||
time_taken = end - start
|
||||
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||||
|
||||
result_AP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, 'AP')
|
||||
result_cAP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, 'cAP')
|
||||
logger.info(f'per_class_ap: {result_AP["per_class_AP"]}')
|
||||
logger.info(f'all_class_names: {self.all_class_names}')
|
||||
logger.info(f'len(result_AP["per_class_AP"]: {len(result_AP["per_class_AP"])}')
|
||||
|
||||
per_class_AP = result_AP["per_class_AP"]
|
||||
if use_wandb == True:
|
||||
wandb.log({"Validation Loss": eval_loss / len(dataloader)})
|
||||
wandb.log({"Validation per_class_AP": per_class_AP})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Validation Loss", eval_loss / len(dataloader), epoch)
|
||||
|
||||
sum_class_AP = 0.0
|
||||
|
||||
for class_name in self.all_class_names:
|
||||
# TVSeries
|
||||
if class_name == 'background':
|
||||
continue
|
||||
# THUMOS
|
||||
if class_name == 'Background' or class_name == 'Ambiguous':
|
||||
continue
|
||||
|
||||
# logger.info(f'{class_name}')
|
||||
class_AP = result_AP["per_class_AP"][class_name]
|
||||
sum_class_AP += class_AP
|
||||
|
||||
class_cAP = result_cAP["per_class_AP"][class_name]
|
||||
logger.info(f'AP @ {class_name}: {class_AP:.4f}')
|
||||
logger.info(f'cAP @ {class_name}: {class_cAP:.4f}')
|
||||
|
||||
logger.info(f'sum_AP: {sum_class_AP}, mean_AP: {sum_class_AP / (len(self.all_class_names) - 1):.4f}')
|
||||
|
||||
logger.info(f'pred_scores: {pred_scores[:2]}')
|
||||
logger.info(f'len(pred_scores): {len(pred_scores)}')
|
||||
logger.info(f'len(pred_scores[0]): {len(pred_scores[0])}')
|
||||
|
||||
return result_AP['mean_AP'], result_cAP['mean_AP'], eval_preds
|
||||
|
||||
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None, result_path = None):
|
||||
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path)
|
||||
|
||||
@EVAL.register("OADTR")
|
||||
class EvaluateOADTR(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(EvaluateOADTR, self).__init__()
|
||||
self.data_processing = thumos_postprocessing if 'THUMOS' in cfg['data_name'] else None
|
||||
self.metric = cfg['metric']
|
||||
self.eval_method = perframe_average_precision
|
||||
self.all_class_names = json.load(open(cfg['video_list_path']))[cfg["data_name"].split('_')[0]]['class_index']
|
||||
self.cfg = cfg
|
||||
|
||||
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer):
|
||||
device = "cuda:0"
|
||||
model.eval()
|
||||
eval_preds = {}
|
||||
gt_targets_dict = {}
|
||||
|
||||
with torch.no_grad():
|
||||
pred_scores, gt_targets = [], []
|
||||
start = time.time()
|
||||
eval_loss = 0.0
|
||||
for it, (vid_name, rgb_input, flow_input, enc_target, distance_target, class_h_target, dec_target) in enumerate(tqdm(dataloader, desc='Evaluation:', leave=False)):
|
||||
rgb_input, flow_input, target = rgb_input.to(device), flow_input.to(device), target.to(device)
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
|
||||
if criterion != None:
|
||||
loss = criterion(out_dict, target)
|
||||
eval_loss += loss.item()
|
||||
|
||||
pred_logit = out_dict['logits']
|
||||
prob_val = pred_logit.squeeze().cpu().numpy()
|
||||
target_batch = target.squeeze().cpu().numpy()
|
||||
pred_scores += list(prob_val)
|
||||
gt_targets += list(target_batch)
|
||||
|
||||
# eval_preds[vid_name] = pred_scores
|
||||
# gt_targets_dict[vid_name] = pred_scores
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({"Validation Loss": eval_loss / len(dataloader)})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Validation Loss", eval_loss / len(dataloader), epoch)
|
||||
|
||||
end = time.time()
|
||||
num_frames = len(gt_targets)
|
||||
result_AP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.data_processing, 'AP')
|
||||
result_cAP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.data_processing, 'cAP')
|
||||
time_taken = end - start
|
||||
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||||
# logger.info(f'per_class_ap: {result["per_class_AP"]}')
|
||||
logger.info(f'all_class_names: {self.all_class_names}')
|
||||
logger.info(f'len(result_AP["per_class_AP"]: {len(result_AP["per_class_AP"])}')
|
||||
|
||||
sum_class_AP = 0.0
|
||||
|
||||
for class_name in self.all_class_names:
|
||||
# TVSeries
|
||||
if class_name == 'background':
|
||||
continue
|
||||
# THUMOS
|
||||
if class_name == 'Background' or class_name == 'Ambiguous':
|
||||
continue
|
||||
|
||||
# logger.info(f'{class_name}')
|
||||
class_AP = result_AP["per_class_AP"][class_name]
|
||||
sum_class_AP += class_AP
|
||||
|
||||
class_cAP = result_cAP["per_class_AP"][class_name]
|
||||
logger.info(f'AP @ {class_name}: {class_AP:.4f}')
|
||||
logger.info(f'cAP @ {class_name}: {class_cAP:.4f}')
|
||||
|
||||
logger.info(f'sum_AP: {sum_class_AP}, mean_AP: {sum_class_AP / (len(self.all_class_names) - 1):.4f}')
|
||||
|
||||
logger.info(f'pred_scores: {pred_scores[:2]}')
|
||||
logger.info(f'len(pred_scores): {len(pred_scores)}')
|
||||
logger.info(f'len(pred_scores[0]): {len(pred_scores[0])}')
|
||||
|
||||
return result_AP['mean_AP'], result_cAP['mean_AP'], eval_preds
|
||||
|
||||
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None):
|
||||
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer)
|
||||
|
||||
|
||||
@EVAL.register("ANTICIPATION")
|
||||
class ANT_Evaluate(nn.Module):
|
||||
|
||||
def __init__(self, cfg):
|
||||
super(ANT_Evaluate, self).__init__()
|
||||
data_name = cfg["data_name"].split('_')[0]
|
||||
self.data_processing = thumos_postprocessing if data_name == 'THUMOS' else None
|
||||
self.metric = cfg['metric']
|
||||
self.eval_method = perframe_average_precision
|
||||
self.event_eval_method = event_accuracy
|
||||
self.all_class_names = json.load(open(cfg['video_list_path']))[data_name]['class_index']
|
||||
self.cfg = cfg
|
||||
|
||||
if 'ignore_index' in cfg:
|
||||
self.ignore_index = cfg['ignore_index']
|
||||
else:
|
||||
raise NotImplementedError("Please set ignore_index in config")
|
||||
|
||||
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path):
|
||||
device = "cuda:0"
|
||||
model.eval()
|
||||
ant_eval_pred_scores_dict = {}
|
||||
ant_gt_targets_dict = {}
|
||||
|
||||
with torch.no_grad():
|
||||
pred_scores, gt_targets, ant_pred_scores, ant_gt_targets = [], [], [], []
|
||||
start = time.time()
|
||||
anticipation_mAPs = []
|
||||
for vid_name, rgb_input, flow_input, target, ant_target in tqdm(dataloader, desc='Evaluation:', leave=False):
|
||||
rgb_input, flow_input, target, ant_target = rgb_input.to(device), flow_input.to(device), target.to(device), ant_target.to(device)
|
||||
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
pred_logit = out_dict['logits']
|
||||
ant_pred_logit = out_dict['anticipation_logits']
|
||||
|
||||
prob_val = pred_logit.squeeze().cpu().numpy() # [8390, 3]
|
||||
target_batch = target.squeeze().cpu().numpy()
|
||||
ant_prob_val = ant_pred_logit.squeeze().cpu().numpy() # [8390, ANTICIPATION_LENGTH, 3]
|
||||
ant_target_batch = ant_target.squeeze().cpu().numpy()
|
||||
|
||||
pred_scores += list(prob_val)
|
||||
gt_targets += list(target_batch)
|
||||
ant_pred_scores += list(ant_prob_val)
|
||||
ant_gt_targets += list(ant_target_batch)
|
||||
if isinstance(vid_name, tuple):
|
||||
vid_name = vid_name[0]
|
||||
|
||||
ant_eval_pred_scores_dict[vid_name] = ant_prob_val
|
||||
ant_gt_targets_dict[vid_name] = ant_target_batch
|
||||
|
||||
end = time.time()
|
||||
num_frames = len(gt_targets)
|
||||
|
||||
result = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, self.metric)
|
||||
per_class_AP = result['per_class_AP']
|
||||
|
||||
ant_pred_scores = np.array(ant_pred_scores)
|
||||
ant_gt_targets = np.array(ant_gt_targets)
|
||||
logger.info(f'OAD mAP: {result["mean_AP"]*100:.2f}')
|
||||
# Prepare the formatted string
|
||||
formatted_output = "per_class_AP:\n" + "\n".join(f" {class_name:<25} : {ap_value:.4f}" for class_name, ap_value in per_class_AP.items())
|
||||
|
||||
# Log the formatted string
|
||||
logger.info(formatted_output)
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({"OAD mAP": result["mean_AP"]})
|
||||
|
||||
anticipation_mAPs_per_class = {key: [] for key in per_class_AP.keys()}
|
||||
|
||||
for step in range(ant_gt_targets.shape[1]):
|
||||
result[f'anticipation_{step+1}'] = self.eval_method(ant_pred_scores[:,step,:], ant_gt_targets[:,step,:], self.all_class_names, self.ignore_index, self.data_processing, self.metric)
|
||||
anticipation_mAPs.append(result[f'anticipation_{step+1}']['mean_AP'])
|
||||
formatted_output = "per_class_AP:\n" + "\n".join(f" {class_name:<25} : {ap_value:.4f}" for class_name, ap_value in result[f'anticipation_{step+1}']['per_class_AP'].items())
|
||||
logger.info(f"Anticipation at step {step+1}: {result[f'anticipation_{step+1}']['mean_AP']*100:.2f}")
|
||||
logger.info(formatted_output)
|
||||
|
||||
for class_name in self.all_class_names:
|
||||
anticipation_mAPs_per_class[class_name].append(result[f'anticipation_{step+1}']['per_class_AP'][class_name])
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({f"Anticipation at step {step+1}": result[f'anticipation_{step+1}']['mean_AP']})
|
||||
|
||||
logger.info(f'Mean Anticipation mAP: {np.mean(anticipation_mAPs)*100:.2f}')
|
||||
formatted_output = "Mean Anticipation per_class_AP:\n" + "\n".join(f" {class_name:<25} : {np.mean(anticipation_mAPs_per_class[class_name])*100:.4f}" for class_name in self.all_class_names)
|
||||
# logger.info(f'Mean Anticipation mAP per class: {np.mean(list(anticipation_mAPs_per_class.values()), axis=1)}')
|
||||
logger.info(formatted_output)
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({"Mean Anticipation mAP": np.mean(anticipation_mAPs)})
|
||||
|
||||
time_taken = end - start
|
||||
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||||
|
||||
return np.mean(anticipation_mAPs), np.mean(anticipation_mAPs), ant_eval_pred_scores_dict
|
||||
|
||||
# def forward(self, model, dataloader, logger):
|
||||
# return self.eval(model, dataloader, logger)
|
||||
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None, result_path=None):
|
||||
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path)
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
__all__ = [
|
||||
'build_eval'
|
||||
]
|
||||
|
||||
from utils import Registry
|
||||
|
||||
EVAL = Registry()
|
||||
|
||||
def build_eval(cfg):
|
||||
eval = EVAL[cfg["task"]](cfg)
|
||||
return eval
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
import torch
|
||||
import wandb
|
||||
from tqdm import tqdm
|
||||
from trainer.train_builder import TRAINER
|
||||
|
||||
|
||||
@TRAINER.register("OAD")
|
||||
def train_one_epoch(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, target = rgb_input.cuda(), flow_input.cuda(), target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
# scaler is None as argparse by default
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
return epoch_loss
|
||||
|
||||
@TRAINER.register("OADTR")
|
||||
def train_one_epoch_oadtr(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, enc_target, distance_target, class_h_target, dec_target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, enc_target, distance_target, class_h_target = rgb_input.cuda(), flow_input.cuda(), enc_target.cuda(), distance_target.cuda(), class_h_target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, enc_target, dec_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, enc_target, dec_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
epoch_loss += loss.item()
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
|
||||
return epoch_loss
|
||||
|
||||
|
||||
@TRAINER.register("ANTICIPATION")
|
||||
def ant_train_one_epoch(trainloader, model, criterion, optimizer, scaler, epoch, use_wandb = False, writer=None, scheduler=None):
|
||||
epoch_loss = 0
|
||||
for it, (vid_name, rgb_input, flow_input, target, ant_target) in enumerate(tqdm(trainloader, desc=f'Epoch:{epoch} Training', postfix=f'lr: {optimizer.param_groups[0]["lr"]:.7f}')):
|
||||
rgb_input, flow_input, target, ant_target = rgb_input.cuda(), flow_input.cuda(), target.cuda(), ant_target.cuda()
|
||||
model.train()
|
||||
if scaler != None:
|
||||
with torch.cuda.amp.autocast():
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target, ant_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
scaler.scale(loss).backward()
|
||||
scaler.step(optimizer)
|
||||
scaler.update()
|
||||
else:
|
||||
out_dict = model(rgb_input, flow_input)
|
||||
loss = criterion(out_dict, target, ant_target)
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
epoch_loss += loss.item()
|
||||
|
||||
if use_wandb == True:
|
||||
wandb.log({"Train Loss": loss.item()})
|
||||
|
||||
if writer != None:
|
||||
writer.add_scalar("Train Loss", loss.item(), it+epoch*len(trainloader))
|
||||
|
||||
return epoch_loss
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
__all__ = [
|
||||
'build_trainer'
|
||||
]
|
||||
|
||||
from utils import Registry
|
||||
|
||||
TRAINER = Registry()
|
||||
|
||||
def build_trainer(cfg):
|
||||
trainer = TRAINER[cfg["task"]]
|
||||
return trainer
|
||||
Reference in New Issue
Block a user