Add causal person fusion and MuVAP stage-3 baselines
This commit is contained in:
Xin Wang
+162
@@ -0,0 +1,162 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
import pandas as pd
|
||||
import os.path as osp
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
from external.rulstm.RULSTM.utils import (get_marginal_indexes, marginalize, softmax,
|
||||
topk_accuracy_multiple_timesteps,
|
||||
topk_recall_multiple_timesteps,
|
||||
tta)
|
||||
|
||||
|
||||
def eval_fusion(pred_scores_files, weights):
|
||||
### Comments: Per-frame scores cost too much space...
|
||||
# perframe_gt_targets_fusion = {}
|
||||
# perframe_pred_scores_fusion = {}
|
||||
# for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
# pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
# cfg = pred_scores['cfg']
|
||||
# assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
# perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
# perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# for session in perframe_pred_scores:
|
||||
# if session not in perframe_pred_scores_fusion:
|
||||
# perframe_pred_scores_fusion[session] = weight * perframe_pred_scores[session]
|
||||
# else:
|
||||
# perframe_pred_scores_fusion[session] += weight * perframe_pred_scores[session]
|
||||
# perframe_gt_targets_fusion[session] = perframe_gt_targets[session]
|
||||
|
||||
# # Compute results
|
||||
# result = compute_result['perframe'](
|
||||
# cfg,
|
||||
# np.concatenate(list(perframe_gt_targets_fusion.values()), axis=0),
|
||||
# np.concatenate(list(perframe_pred_scores_fusion.values()), axis=0),
|
||||
# )
|
||||
# logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
# cfg.DATA.METRICS, result['mean_AP']
|
||||
# ))
|
||||
|
||||
action_scores = None
|
||||
for weight, pred_scores_file in zip(weights, pred_scores_files):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
assert cfg.DATA.DATA_NAME in ['EK55', 'EK100']
|
||||
if action_scores is None:
|
||||
action_scores = weight * pred_scores['action_scores']
|
||||
else:
|
||||
action_scores += weight * pred_scores['action_scores']
|
||||
# segment-level evaluation on EK55/EK100
|
||||
if cfg.DATA.DATA_NAME == 'EK55':
|
||||
path_to_data = 'external/rulstm/RULSTM/data/ek55/'
|
||||
segment_list = pd.read_csv(osp.join(path_to_data, 'validation.csv'),
|
||||
names=['id', 'video', 'start_f', 'end_f', 'verb', 'noun', 'action'],
|
||||
skipinitialspace=True)
|
||||
predictions, labels, ids = [], [], []
|
||||
discarded_labels, discarded_ids = [], []
|
||||
for idx in segment_list['id']:
|
||||
segment = segment_list[segment_list['id'] == idx]
|
||||
start_step = int(np.floor(segment['start_f'].values.astype(int) / 30 * cfg.DATA.FPS))
|
||||
vid = segment['video'].values[0].strip()
|
||||
# preds = (perframe_pred_scores_fusion[vid][start_step][None, 1:, ::-1]).transpose(0, 2, 1)
|
||||
if start_step >= cfg.MODEL.LSTR.ANTICIPATION_LENGTH:
|
||||
# predictions.append(preds)
|
||||
labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
ids.append(idx)
|
||||
else:
|
||||
discarded_labels.append(segment[['verb', 'noun', 'action']].values.astype(int))
|
||||
discarded_ids.append(idx)
|
||||
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'), index_col='id')
|
||||
|
||||
# action_scores = np.concatenate(predictions)
|
||||
labels = np.concatenate(labels)
|
||||
ids = np.array(ids)
|
||||
|
||||
vi = get_marginal_indexes(actions, 'verb')
|
||||
ni = get_marginal_indexes(actions, 'noun')
|
||||
|
||||
action_probs = softmax(action_scores.reshape(-1, action_scores.shape[-1]))
|
||||
# action_probs = action_scores.reshape(-1, action_scores.shape[-1])
|
||||
# action_probs = action_probs / action_probs.sum(-1, keepdims=True)
|
||||
print(action_probs.shape)
|
||||
verb_scores = marginalize(action_probs, vi).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
noun_scores = marginalize(action_probs, ni).reshape(action_scores.shape[0],
|
||||
action_scores.shape[1],
|
||||
-1)
|
||||
|
||||
include_discarded = (cfg.DATA.DATA_NAME == 'EK100')
|
||||
if include_discarded:
|
||||
dlab = np.array(discarded_labels)
|
||||
dislab = np.array(discarded_ids)
|
||||
ids = np.concatenate([ids, dislab])
|
||||
num_disc = len(dlab)
|
||||
labels = np.concatenate([labels, dlab])
|
||||
verb_scores = np.concatenate((verb_scores, np.zeros((num_disc, *verb_scores.shape[1:]))))
|
||||
noun_scores = np.concatenate((noun_scores, np.zeros((num_disc, *noun_scores.shape[1:]))))
|
||||
action_scores = np.concatenate((action_scores, np.zeros((num_disc, *action_scores.shape[1:]))))
|
||||
|
||||
verb_labels, noun_labels, action_labels = labels[:, 0], labels[:, 1], labels[:, 2]
|
||||
verb_accuracies = topk_accuracy_multiple_timesteps(verb_scores, verb_labels)
|
||||
noun_accuracies = topk_accuracy_multiple_timesteps(noun_scores, noun_labels)
|
||||
action_accuracies = topk_accuracy_multiple_timesteps(action_scores, action_labels)
|
||||
|
||||
many_shot_verbs = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_verbs.csv'))['verb_class'].values
|
||||
many_shot_nouns = pd.read_csv(osp.join(path_to_data, 'EPIC_many_shot_nouns.csv'))['noun_class'].values
|
||||
actions = pd.read_csv(osp.join(path_to_data, 'actions.csv'))
|
||||
a_to_vn = {a[1]['id']: tuple(a[1][['verb', 'noun']].values)
|
||||
for a in actions.iterrows()}
|
||||
many_shot_actions = [a for a, (v, n) in a_to_vn.items()
|
||||
if v in many_shot_verbs or n in many_shot_nouns]
|
||||
|
||||
verb_recalls = topk_recall_multiple_timesteps(verb_scores, verb_labels,
|
||||
k=5, classes=many_shot_verbs)
|
||||
noun_recalls = topk_recall_multiple_timesteps(noun_scores, noun_labels,
|
||||
k=5, classes=many_shot_nouns)
|
||||
action_recalls = topk_recall_multiple_timesteps(action_scores, action_labels,
|
||||
k=5, classes=many_shot_actions)
|
||||
|
||||
all_accuracies = np.concatenate([verb_accuracies, noun_accuracies, action_accuracies,
|
||||
verb_recalls, noun_recalls, action_recalls])
|
||||
all_accuracies = all_accuracies[[0, 1, 6, 2, 3, 7, 4, 5, 8]]
|
||||
indices = [
|
||||
('Verb', 'Top-1 Accuracy'), ('Verb', 'Top-5 Accuracy'),
|
||||
('Verb', 'Mean Top-5 Recall'),
|
||||
('Noun', 'Top-1 Accuracy'), ('Noun', 'Top-5 Accuracy'),
|
||||
('Noun', 'Mean Top-5 Recall'),
|
||||
('Action', 'Top-1 Accuracy'), ('Action', 'Top-5 Accuracy'),
|
||||
('Action', 'Mean Top-5 Recall'),
|
||||
]
|
||||
cc = np.linspace(1 / cfg.DATA.FPS * cfg.MODEL.LSTR.ANTICIPATION_LENGTH,
|
||||
1 / cfg.DATA.FPS, cfg.MODEL.LSTR.ANTICIPATION_LENGTH, dtype=str)
|
||||
scores = pd.DataFrame(all_accuracies * 100, columns=cc,
|
||||
index=pd.MultiIndex.from_tuples(indices))
|
||||
print(scores)
|
||||
|
||||
tta_verb = tta(verb_scores, verb_labels)
|
||||
tta_noun = tta(noun_scores, noun_labels)
|
||||
tta_action = tta(action_scores, action_labels)
|
||||
print(f'\nMean TtA(5): VERB {tta_verb:0.2f} '
|
||||
f'NOUN: {tta_noun:0.2f} ACTION: {tta_action:0.2f}')
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_files', nargs='+', required=True)
|
||||
parser.add_argument('--weights', type=float, nargs='+', required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
assert len(args.pred_scores_files) == len(args.weights)
|
||||
eval_fusion(args.pred_scores_files, args.weights)
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perframe(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perframe'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
logging.info('Action detection perframe m{}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, result['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perframe(args.pred_scores_file)
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(message)s')
|
||||
import argparse
|
||||
|
||||
import numpy as np
|
||||
import pickle as pkl
|
||||
|
||||
from rekognition_online_action_detection.evaluation import compute_result
|
||||
|
||||
|
||||
def eval_perstage(pred_scores_file):
|
||||
pred_scores = pkl.load(open(pred_scores_file, 'rb'))
|
||||
cfg = pred_scores['cfg']
|
||||
perframe_gt_targets = pred_scores['perframe_gt_targets']
|
||||
perframe_pred_scores = pred_scores['perframe_pred_scores']
|
||||
|
||||
# Compute results
|
||||
result = compute_result['perstage'](
|
||||
cfg,
|
||||
np.concatenate(list(perframe_gt_targets.values()), axis=0),
|
||||
np.concatenate(list(perframe_pred_scores.values()), axis=0),
|
||||
)
|
||||
for stage_name in result:
|
||||
logging.info('Perframe m{} of stage {}: {:.5f}'.format(
|
||||
cfg.DATA.METRICS, stage_name, result[stage_name]['mean_AP']
|
||||
))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pred_scores_file', type=str, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
eval_perstage(args.pred_scores_file)
|
||||
Reference in New Issue
Block a user