289 lines
14 KiB
Python
289 lines
14 KiB
Python
|
import json
|
||
|
|
import time
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import torch
|
||
|
|
import torch.nn as nn
|
||
|
|
import wandb
|
||
|
|
from tqdm import tqdm
|
||
|
|
from trainer.eval_builder import EVAL
|
||
|
|
from utils import *
|
||
|
|
from utils import (event_accuracy, perframe_average_precision,
|
||
|
|
thumos_postprocessing)
|
||
|
|
|
||
|
|
|
||
|
|
@EVAL.register("OAD")
|
||
|
|
class Evaluate(nn.Module):
|
||
|
|
|
||
|
|
def __init__(self, cfg):
|
||
|
|
super(Evaluate, self).__init__()
|
||
|
|
self.data_processing = thumos_postprocessing if 'THUMOS' in cfg['data_name'] else None
|
||
|
|
self.metric = cfg['metric']
|
||
|
|
self.eval_method = perframe_average_precision
|
||
|
|
self.event_eval_method = event_accuracy
|
||
|
|
self.all_class_names = json.load(open(cfg['video_list_path']))[cfg["data_name"].split('_')[0]]['class_index']
|
||
|
|
self.cfg = cfg
|
||
|
|
self.ignore_index = cfg['ignore_index'] if 'ignore_index' in cfg else -100
|
||
|
|
|
||
|
|
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path):
|
||
|
|
device = "cuda:0"
|
||
|
|
model.eval()
|
||
|
|
eval_preds = {}
|
||
|
|
gt_targets_dict = {}
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
pred_scores, gt_targets = [], []
|
||
|
|
start = time.time()
|
||
|
|
eval_loss = 0.0
|
||
|
|
for it, (vid_name, rgb_input, flow_input, target) in enumerate(tqdm(dataloader, desc='Evaluation:', leave=False)):
|
||
|
|
rgb_input, flow_input, target = rgb_input.to(device), flow_input.to(device), target.to(device)
|
||
|
|
out_dict = model(rgb_input, flow_input)
|
||
|
|
|
||
|
|
if criterion != None:
|
||
|
|
loss = criterion(out_dict, target)
|
||
|
|
eval_loss += loss.item()
|
||
|
|
|
||
|
|
pred_logit = out_dict['logits']
|
||
|
|
prob_val = pred_logit.squeeze().cpu().numpy()
|
||
|
|
target_batch = target.squeeze().cpu().numpy()
|
||
|
|
pred_scores += list(prob_val)
|
||
|
|
gt_targets += list(target_batch)
|
||
|
|
|
||
|
|
eval_preds[vid_name] = prob_val
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
num_frames = len(gt_targets)
|
||
|
|
|
||
|
|
# save prediction scores
|
||
|
|
# if result_path != None:
|
||
|
|
time_taken = end - start
|
||
|
|
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||
|
|
|
||
|
|
result_AP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, 'AP')
|
||
|
|
result_cAP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, 'cAP')
|
||
|
|
logger.info(f'per_class_ap: {result_AP["per_class_AP"]}')
|
||
|
|
logger.info(f'all_class_names: {self.all_class_names}')
|
||
|
|
logger.info(f'len(result_AP["per_class_AP"]: {len(result_AP["per_class_AP"])}')
|
||
|
|
|
||
|
|
per_class_AP = result_AP["per_class_AP"]
|
||
|
|
if use_wandb == True:
|
||
|
|
wandb.log({"Validation Loss": eval_loss / len(dataloader)})
|
||
|
|
wandb.log({"Validation per_class_AP": per_class_AP})
|
||
|
|
|
||
|
|
if writer != None:
|
||
|
|
writer.add_scalar("Validation Loss", eval_loss / len(dataloader), epoch)
|
||
|
|
|
||
|
|
sum_class_AP = 0.0
|
||
|
|
|
||
|
|
for class_name in self.all_class_names:
|
||
|
|
# TVSeries
|
||
|
|
if class_name == 'background':
|
||
|
|
continue
|
||
|
|
# THUMOS
|
||
|
|
if class_name == 'Background' or class_name == 'Ambiguous':
|
||
|
|
continue
|
||
|
|
|
||
|
|
# logger.info(f'{class_name}')
|
||
|
|
class_AP = result_AP["per_class_AP"][class_name]
|
||
|
|
sum_class_AP += class_AP
|
||
|
|
|
||
|
|
class_cAP = result_cAP["per_class_AP"][class_name]
|
||
|
|
logger.info(f'AP @ {class_name}: {class_AP:.4f}')
|
||
|
|
logger.info(f'cAP @ {class_name}: {class_cAP:.4f}')
|
||
|
|
|
||
|
|
logger.info(f'sum_AP: {sum_class_AP}, mean_AP: {sum_class_AP / (len(self.all_class_names) - 1):.4f}')
|
||
|
|
|
||
|
|
logger.info(f'pred_scores: {pred_scores[:2]}')
|
||
|
|
logger.info(f'len(pred_scores): {len(pred_scores)}')
|
||
|
|
logger.info(f'len(pred_scores[0]): {len(pred_scores[0])}')
|
||
|
|
|
||
|
|
return result_AP['mean_AP'], result_cAP['mean_AP'], eval_preds
|
||
|
|
|
||
|
|
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None, result_path = None):
|
||
|
|
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path)
|
||
|
|
|
||
|
|
@EVAL.register("OADTR")
|
||
|
|
class EvaluateOADTR(nn.Module):
|
||
|
|
|
||
|
|
def __init__(self, cfg):
|
||
|
|
super(EvaluateOADTR, self).__init__()
|
||
|
|
self.data_processing = thumos_postprocessing if 'THUMOS' in cfg['data_name'] else None
|
||
|
|
self.metric = cfg['metric']
|
||
|
|
self.eval_method = perframe_average_precision
|
||
|
|
self.all_class_names = json.load(open(cfg['video_list_path']))[cfg["data_name"].split('_')[0]]['class_index']
|
||
|
|
self.cfg = cfg
|
||
|
|
|
||
|
|
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer):
|
||
|
|
device = "cuda:0"
|
||
|
|
model.eval()
|
||
|
|
eval_preds = {}
|
||
|
|
gt_targets_dict = {}
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
pred_scores, gt_targets = [], []
|
||
|
|
start = time.time()
|
||
|
|
eval_loss = 0.0
|
||
|
|
for it, (vid_name, rgb_input, flow_input, enc_target, distance_target, class_h_target, dec_target) in enumerate(tqdm(dataloader, desc='Evaluation:', leave=False)):
|
||
|
|
rgb_input, flow_input, target = rgb_input.to(device), flow_input.to(device), target.to(device)
|
||
|
|
out_dict = model(rgb_input, flow_input)
|
||
|
|
|
||
|
|
if criterion != None:
|
||
|
|
loss = criterion(out_dict, target)
|
||
|
|
eval_loss += loss.item()
|
||
|
|
|
||
|
|
pred_logit = out_dict['logits']
|
||
|
|
prob_val = pred_logit.squeeze().cpu().numpy()
|
||
|
|
target_batch = target.squeeze().cpu().numpy()
|
||
|
|
pred_scores += list(prob_val)
|
||
|
|
gt_targets += list(target_batch)
|
||
|
|
|
||
|
|
# eval_preds[vid_name] = pred_scores
|
||
|
|
# gt_targets_dict[vid_name] = pred_scores
|
||
|
|
|
||
|
|
if use_wandb == True:
|
||
|
|
wandb.log({"Validation Loss": eval_loss / len(dataloader)})
|
||
|
|
|
||
|
|
if writer != None:
|
||
|
|
writer.add_scalar("Validation Loss", eval_loss / len(dataloader), epoch)
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
num_frames = len(gt_targets)
|
||
|
|
result_AP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.data_processing, 'AP')
|
||
|
|
result_cAP = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.data_processing, 'cAP')
|
||
|
|
time_taken = end - start
|
||
|
|
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||
|
|
# logger.info(f'per_class_ap: {result["per_class_AP"]}')
|
||
|
|
logger.info(f'all_class_names: {self.all_class_names}')
|
||
|
|
logger.info(f'len(result_AP["per_class_AP"]: {len(result_AP["per_class_AP"])}')
|
||
|
|
|
||
|
|
sum_class_AP = 0.0
|
||
|
|
|
||
|
|
for class_name in self.all_class_names:
|
||
|
|
# TVSeries
|
||
|
|
if class_name == 'background':
|
||
|
|
continue
|
||
|
|
# THUMOS
|
||
|
|
if class_name == 'Background' or class_name == 'Ambiguous':
|
||
|
|
continue
|
||
|
|
|
||
|
|
# logger.info(f'{class_name}')
|
||
|
|
class_AP = result_AP["per_class_AP"][class_name]
|
||
|
|
sum_class_AP += class_AP
|
||
|
|
|
||
|
|
class_cAP = result_cAP["per_class_AP"][class_name]
|
||
|
|
logger.info(f'AP @ {class_name}: {class_AP:.4f}')
|
||
|
|
logger.info(f'cAP @ {class_name}: {class_cAP:.4f}')
|
||
|
|
|
||
|
|
logger.info(f'sum_AP: {sum_class_AP}, mean_AP: {sum_class_AP / (len(self.all_class_names) - 1):.4f}')
|
||
|
|
|
||
|
|
logger.info(f'pred_scores: {pred_scores[:2]}')
|
||
|
|
logger.info(f'len(pred_scores): {len(pred_scores)}')
|
||
|
|
logger.info(f'len(pred_scores[0]): {len(pred_scores[0])}')
|
||
|
|
|
||
|
|
return result_AP['mean_AP'], result_cAP['mean_AP'], eval_preds
|
||
|
|
|
||
|
|
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None):
|
||
|
|
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer)
|
||
|
|
|
||
|
|
|
||
|
|
@EVAL.register("ANTICIPATION")
|
||
|
|
class ANT_Evaluate(nn.Module):
|
||
|
|
|
||
|
|
def __init__(self, cfg):
|
||
|
|
super(ANT_Evaluate, self).__init__()
|
||
|
|
data_name = cfg["data_name"].split('_')[0]
|
||
|
|
self.data_processing = thumos_postprocessing if data_name == 'THUMOS' else None
|
||
|
|
self.metric = cfg['metric']
|
||
|
|
self.eval_method = perframe_average_precision
|
||
|
|
self.event_eval_method = event_accuracy
|
||
|
|
self.all_class_names = json.load(open(cfg['video_list_path']))[data_name]['class_index']
|
||
|
|
self.cfg = cfg
|
||
|
|
|
||
|
|
if 'ignore_index' in cfg:
|
||
|
|
self.ignore_index = cfg['ignore_index']
|
||
|
|
else:
|
||
|
|
raise NotImplementedError("Please set ignore_index in config")
|
||
|
|
|
||
|
|
def eval(self, model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path):
|
||
|
|
device = "cuda:0"
|
||
|
|
model.eval()
|
||
|
|
ant_eval_pred_scores_dict = {}
|
||
|
|
ant_gt_targets_dict = {}
|
||
|
|
|
||
|
|
with torch.no_grad():
|
||
|
|
pred_scores, gt_targets, ant_pred_scores, ant_gt_targets = [], [], [], []
|
||
|
|
start = time.time()
|
||
|
|
anticipation_mAPs = []
|
||
|
|
for vid_name, rgb_input, flow_input, target, ant_target in tqdm(dataloader, desc='Evaluation:', leave=False):
|
||
|
|
rgb_input, flow_input, target, ant_target = rgb_input.to(device), flow_input.to(device), target.to(device), ant_target.to(device)
|
||
|
|
|
||
|
|
out_dict = model(rgb_input, flow_input)
|
||
|
|
pred_logit = out_dict['logits']
|
||
|
|
ant_pred_logit = out_dict['anticipation_logits']
|
||
|
|
|
||
|
|
prob_val = pred_logit.squeeze().cpu().numpy() # [8390, 3]
|
||
|
|
target_batch = target.squeeze().cpu().numpy()
|
||
|
|
ant_prob_val = ant_pred_logit.squeeze().cpu().numpy() # [8390, ANTICIPATION_LENGTH, 3]
|
||
|
|
ant_target_batch = ant_target.squeeze().cpu().numpy()
|
||
|
|
|
||
|
|
pred_scores += list(prob_val)
|
||
|
|
gt_targets += list(target_batch)
|
||
|
|
ant_pred_scores += list(ant_prob_val)
|
||
|
|
ant_gt_targets += list(ant_target_batch)
|
||
|
|
if isinstance(vid_name, tuple):
|
||
|
|
vid_name = vid_name[0]
|
||
|
|
|
||
|
|
ant_eval_pred_scores_dict[vid_name] = ant_prob_val
|
||
|
|
ant_gt_targets_dict[vid_name] = ant_target_batch
|
||
|
|
|
||
|
|
end = time.time()
|
||
|
|
num_frames = len(gt_targets)
|
||
|
|
|
||
|
|
result = self.eval_method(pred_scores, gt_targets, self.all_class_names, self.ignore_index, self.data_processing, self.metric)
|
||
|
|
per_class_AP = result['per_class_AP']
|
||
|
|
|
||
|
|
ant_pred_scores = np.array(ant_pred_scores)
|
||
|
|
ant_gt_targets = np.array(ant_gt_targets)
|
||
|
|
logger.info(f'OAD mAP: {result["mean_AP"]*100:.2f}')
|
||
|
|
# Prepare the formatted string
|
||
|
|
formatted_output = "per_class_AP:\n" + "\n".join(f" {class_name:<25} : {ap_value:.4f}" for class_name, ap_value in per_class_AP.items())
|
||
|
|
|
||
|
|
# Log the formatted string
|
||
|
|
logger.info(formatted_output)
|
||
|
|
|
||
|
|
if use_wandb == True:
|
||
|
|
wandb.log({"OAD mAP": result["mean_AP"]})
|
||
|
|
|
||
|
|
anticipation_mAPs_per_class = {key: [] for key in per_class_AP.keys()}
|
||
|
|
|
||
|
|
for step in range(ant_gt_targets.shape[1]):
|
||
|
|
result[f'anticipation_{step+1}'] = self.eval_method(ant_pred_scores[:,step,:], ant_gt_targets[:,step,:], self.all_class_names, self.ignore_index, self.data_processing, self.metric)
|
||
|
|
anticipation_mAPs.append(result[f'anticipation_{step+1}']['mean_AP'])
|
||
|
|
formatted_output = "per_class_AP:\n" + "\n".join(f" {class_name:<25} : {ap_value:.4f}" for class_name, ap_value in result[f'anticipation_{step+1}']['per_class_AP'].items())
|
||
|
|
logger.info(f"Anticipation at step {step+1}: {result[f'anticipation_{step+1}']['mean_AP']*100:.2f}")
|
||
|
|
logger.info(formatted_output)
|
||
|
|
|
||
|
|
for class_name in self.all_class_names:
|
||
|
|
anticipation_mAPs_per_class[class_name].append(result[f'anticipation_{step+1}']['per_class_AP'][class_name])
|
||
|
|
|
||
|
|
if use_wandb == True:
|
||
|
|
wandb.log({f"Anticipation at step {step+1}": result[f'anticipation_{step+1}']['mean_AP']})
|
||
|
|
|
||
|
|
logger.info(f'Mean Anticipation mAP: {np.mean(anticipation_mAPs)*100:.2f}')
|
||
|
|
formatted_output = "Mean Anticipation per_class_AP:\n" + "\n".join(f" {class_name:<25} : {np.mean(anticipation_mAPs_per_class[class_name])*100:.4f}" for class_name in self.all_class_names)
|
||
|
|
# logger.info(f'Mean Anticipation mAP per class: {np.mean(list(anticipation_mAPs_per_class.values()), axis=1)}')
|
||
|
|
logger.info(formatted_output)
|
||
|
|
|
||
|
|
if use_wandb == True:
|
||
|
|
wandb.log({"Mean Anticipation mAP": np.mean(anticipation_mAPs)})
|
||
|
|
|
||
|
|
time_taken = end - start
|
||
|
|
logger.info(f'Processed {num_frames} frames in {time_taken:.1f} seconds ({num_frames / time_taken :.1f} FPS)')
|
||
|
|
|
||
|
|
return np.mean(anticipation_mAPs), np.mean(anticipation_mAPs), ant_eval_pred_scores_dict
|
||
|
|
|
||
|
|
# def forward(self, model, dataloader, logger):
|
||
|
|
# return self.eval(model, dataloader, logger)
|
||
|
|
def forward(self, model, dataloader, logger, criterion=None, use_wandb=False, epoch=None, writer=None, result_path=None):
|
||
|
|
return self.eval(model, dataloader, logger, criterion, use_wandb, epoch, writer, result_path)
|