import sys import torch import torch.nn as nn import torch.nn.functional as F from configs.feature_sizes import FEATURE_SIZES from model.model_builder import META_ARCHITECTURES from utils.util import parse_flow_type, parse_rgb_type @META_ARCHITECTURES.register("MiniROAD") class MROAD(nn.Module): def __init__(self, cfg): super(MROAD, self).__init__() self.use_flow = not cfg['no_flow'] self.use_rgb = not cfg['no_rgb'] self.input_dim = 0 rgb_type = parse_rgb_type(cfg['rgb_type']) flow_type = parse_flow_type(cfg['flow_type']) if self.use_rgb: self.input_dim += FEATURE_SIZES[rgb_type] if self.use_flow: self.input_dim += FEATURE_SIZES[flow_type] self.hidden_dim = cfg['hidden_dim'] self.num_layers = cfg['num_layers'] self.out_dim = cfg['num_classes'] self.window_size = cfg['window_size'] self.relu = nn.ReLU() self.embedding_dim = cfg['embedding_dim'] self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True) self.layer1 = nn.Sequential( nn.Linear(self.input_dim, self.embedding_dim), nn.LayerNorm(self.embedding_dim), nn.ReLU(), nn.Dropout(p=cfg['dropout']), ) self.f_classification = nn.Sequential( nn.Linear(self.hidden_dim, self.out_dim) ) # self.h0 = torch.nn.Parameter(torch.zeros(1, 1, self.hidden_dim)) self.h0 = torch.zeros(self.num_layers, 1, self.hidden_dim) def forward(self, rgb_input, flow_input): x = torch.empty(0, device='cuda:0') if self.use_rgb: x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension if self.use_flow: x = torch.cat((x, flow_input), dim=2) # if self.use_rgb and self.use_flow: # x = torch.cat((rgb_input, flow_input), 2) # elif self.use_rgb: # x = rgb_input # elif self.use_flow: # x = flow_input x = self.layer1(x) B, _, _ = x.shape h0 = self.h0.expand(-1, B, -1).to(x.device) ht, _ = self.gru(x, h0) ht = self.relu(ht) # ht = self.relu(ht + x) logits = self.f_classification(ht) out_dict = {} if self.training: out_dict['logits'] = logits else: pred_scores = F.softmax(logits, dim=-1) out_dict['logits'] = pred_scores return out_dict @META_ARCHITECTURES.register("MiniROADA") class MROADA(nn.Module): def __init__(self, cfg): super(MROADA, self).__init__() self.use_flow = not cfg['no_flow'] self.use_rgb = not cfg['no_rgb'] self.input_dim = 0 if self.use_rgb: self.input_dim += FEATURE_SIZES[cfg['rgb_type']] if self.use_flow: self.input_dim += FEATURE_SIZES[cfg['flow_type']] self.embedding_dim = cfg['embedding_dim'] self.hidden_dim = cfg['hidden_dim'] self.num_layers = cfg['num_layers'] self.anticipation_length = cfg["anticipation_length"] self.out_dim = cfg['num_classes'] self.layer1 = nn.Sequential( nn.Linear(self.input_dim, self.embedding_dim), nn.LayerNorm(self.embedding_dim), nn.ReLU(), nn.Dropout(p=cfg['dropout']) ) self.actionness = cfg['actionness'] if self.actionness: self.f_actionness = nn.Sequential( nn.Linear(self.hidden_dim, 1), ) self.relu = nn.ReLU() self.gru = nn.GRU(self.embedding_dim, self.hidden_dim, self.num_layers, batch_first=True) self.f_classification = nn.Sequential( nn.Linear(self.hidden_dim, self.out_dim) ) self.anticipation_layer = nn.Sequential( nn.Linear(self.hidden_dim, self.anticipation_length*self.hidden_dim), ) self.cfg = cfg def forward(self, rgb_input, flow_input): x = torch.empty(0, device='cuda:0') if self.use_rgb: # [64, 128, 2048] x = torch.cat((x, rgb_input), dim=2) # Concatenating along the channel dimension if self.use_flow: # [64, 128, 5120] x = torch.cat((x, flow_input), dim=2) # audio channel is too huge to directly concat... # to c B, S, feat_dim = x.shape # batch, length, dim # x = self.layer1(x) h0 = torch.zeros(1, B, self.hidden_dim).to(x.device) ht, _ = self.gru(x, h0) ht = self.relu(ht) logits = self.f_classification(self.relu(ht)) anticipation_ht = self.anticipation_layer(self.relu(ht)).view(B, S, self.anticipation_length, self.hidden_dim) anticipation_logits = self.f_classification(self.relu(anticipation_ht)) out_dict = {} if self.training: out_dict['logits'] = logits out_dict['anticipation_logits'] = anticipation_logits else: pred_scores = F.softmax(logits, dim=-1) pred_anticipation_scores = F.softmax(anticipation_logits, dim=-1) out_dict['logits'] = pred_scores out_dict['anticipation_logits'] = pred_anticipation_scores return out_dict