163 lines
8.1 KiB
Python
163 lines
8.1 KiB
Python
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
import logging
|
|
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
|
import argparse
|
|
|
|
import numpy as np
|
|
import pickle as pkl
|
|
import pandas as pd
|
|
import os.path as osp
|
|
|
|
from rekognition_online_action_detection.evaluation import compute_result
|
|
from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
|
topk_accuracy_multiple_timesteps,
|
|
topk_recall_multiple_timesteps,
|
|
tta)
|
|
|
|
|
|
def eval_fusion(pred_scores_files, weights):
|
|
### Comments: Per-frame scores cost too much space...
|
|
# perframe_gt_targets_fusion = {}
|
|
# perframe_pred_scores_fusion = {}
|
|
# for weight, pred_scores_file in zip(weights, pred_scores_files):
|
|
# pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
|
# cfg = pred_scores['cfg']
|
|
# assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
|
# perframe_gt_targets = pred_scores['perframe_gt_targets']
|
|
# perframe_pred_scores = pred_scores['perframe_pred_scores']
|
|
|
|
# for session in perframe_pred_scores:
|
|
# if session not in perframe_pred_scores_fusion:
|
|
# perframe_pred_scores_fusion[session] = weight * perframe_pred_scores[session]
|
|
# else:
|
|
# perframe_pred_scores_fusion[session] += weight * perframe_pred_scores[session]
|
|
# perframe_gt_targets_fusion[session] = perframe_gt_targets[session]
|
|
|
|
# # Compute results
|
|
# result = compute_result['perframe'](
|
|
# cfg,
|
|
# np.concatenate(list(perframe_gt_targets_fusion.values()), axis=0),
|
|
# np.concatenate(list(perframe_pred_scores_fusion.values()), axis=0),
|
|
# )
|
|
# logging.info('Action detection perframe m{}: {:.5f}'.format(
|
|
# cfg.DATA.METRICS, result['mean_AP']
|
|
# ))
|
|
|
|
action_scores = None
|
|
for weight, pred_scores_file in zip(weights, pred_scores_files):
|
|
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
|
cfg = pred_scores['cfg']
|
|
assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
|
if action_scores is None:
|
|
action_scores = weight * pred_scores['action_scores']
|
|
else:
|
|
action_scores += weight * pred_scores['action_scores']
|
|
# segment-level evaluation on EK55/EK100
|
|
if cfg.DATA.DATA_NAME == 'EK55':
|
|
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
|
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
|
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
|
skipinitialspace=True)
|
|
predictions, labels, ids = [], [], []
|
|
discarded_labels, discarded_ids = [], []
|
|
for idx in segment_list['id']:
|
|
segment = segment_list[segment_list['id'] == idx]
|
|
start_step = int(np.floor(segment['start_f'].values.astype(int) / 30 * cfg.DATA.FPS))
|
|
vid = segment['video'].values[0].strip()
|
|
# preds = (perframe_pred_scores_fusion[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
|
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
|
# predictions.append(preds)
|
|
labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
|
ids.append(idx)
|
|
else:
|
|
discarded_labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
|
discarded_ids.append(idx)
|
|
|
|
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
|
|
|
# action_scores = np.concatenate(predictions)
|
|
labels = np.concatenate(labels)
|
|
ids = np.array(ids)
|
|
|
|
vi = get_marginal_indexes(actions, 'verb')
|
|
ni = get_marginal_indexes(actions, 'noun')
|
|
|
|
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
|
# action_probs = action_scores.reshape(-1, action_scores.shape[-1])
|
|
# action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
|
print(action_probs.shape)
|
|
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
|
action_scores.shape[1],
|
|
-1)
|
|
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
|
action_scores.shape[1],
|
|
-1)
|
|
|
|
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
|
if include_discarded:
|
|
dlab = np.array(discarded_labels)
|
|
dislab = np.array(discarded_ids)
|
|
ids = np.concatenate([ids, dislab])
|
|
num_disc = len(dlab)
|
|
labels = np.concatenate([labels, dlab])
|
|
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
|
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
|
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
|
|
|
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
|
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
|
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
|
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
|
|
|
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
|
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
|
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
|
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
|
for a in actions.iterrows()}
|
|
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
|
if v in many_shot_verbs or n in many_shot_nouns]
|
|
|
|
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
|
k=5, classes=many_shot_verbs)
|
|
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
|
k=5, classes=many_shot_nouns)
|
|
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
|
k=5, classes=many_shot_actions)
|
|
|
|
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
|
verb_recalls, noun_recalls, action_recalls])
|
|
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
|
indices = [
|
|
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
|
('Verb', 'Mean Top-5 Recall'),
|
|
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
|
('Noun', 'Mean Top-5 Recall'),
|
|
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
|
('Action', 'Mean Top-5 Recall'),
|
|
]
|
|
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
|
1 / cfg.DATA.FPS, cfg.MODEL.LSTR.ANTICIPATION_LENGTH, dtype=str)
|
|
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
|
index=pd.MultiIndex.from_tuples(indices))
|
|
print(scores)
|
|
|
|
tta_verb = tta(verb_scores, verb_labels)
|
|
tta_noun = tta(noun_scores, noun_labels)
|
|
tta_action = tta(action_scores, action_labels)
|
|
print(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
|
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
|
else:
|
|
raise NotImplementedError
|
|
|
|
|
|
if __name__ == '__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--pred_scores_files', nargs='+', required=True)
|
|
parser.add_argument('--weights', type=float, nargs='+', required=True)
|
|
args = parser.parse_args()
|
|
|
|
assert len(args.pred_scores_files) == len(args.weights)
|
|
eval_fusion(args.pred_scores_files, args.weights)
|