class_docstr: An implementation of the temporal attention mechanism.
  For details see this paper: `"GMAN: A Graph Multi-Attention Network for Traffic Prediction."
  <https://arxiv.org/pdf/1911.08415.pdf>`_
  
  Args:
      K (int) : Number of attention heads.
      d (int) : Dimension of each attention head outputs.
      bn_decay (float): Batch normalization momentum.
      mask (bool): Whether to mask attention score.
docstr: Making a forward pass of the temporal attention mechanism.
  
  Arg types:
      * **X** (PyTorch Float Tensor) - Input sequence, with shape (batch_size, num_step, num_nodes, K*d).
      * **STE** (Pytorch Float Tensor) - Spatial-temporal embedding, with shape (batch_size, num_step, num_nodes, K*d).
  
  Return types:
      * **X** (PyTorch Float Tensor) - Temporal attention scores, with shape (batch_size, num_step, num_nodes, K*d).
docstr_tok: ["Making","a","forward","pass","of","the","temporal","attention","mechanism",".","Arg","types",":","*","*","*","X","*","*","(","PyTorch","Float","Tensor",")","-","Input","sequence",",","with","shape","(","batch_size",",","num_step",",","num_nodes",",","K","*","d",")",".","*","*","*","STE","*","*","(","Pytorch","Float","Tensor",")","-","Spatial-temporal","embedding",",","with","shape","(","batch_size",",","num_step",",","num_nodes",",","K","*","d",")",".","Return","types",":","*","*","*","X","*","*","(","PyTorch","Float","Tensor",")","-","Temporal","attention","scores",",","with","shape","(","batch_size",",","num_step",",","num_nodes",",","K","*","d",")","."]
code: def forward(
      self, X: torch.FloatTensor, STE: torch.FloatTensor
  ) -> torch.FloatTensor:
      """
      Making a forward pass of the temporal attention mechanism.
  
      Arg types:
          * **X** (PyTorch Float Tensor) - Input sequence, with shape (batch_size, num_step, num_nodes, K*d).
          * **STE** (Pytorch Float Tensor) - Spatial-temporal embedding, with shape (batch_size, num_step, num_nodes, K*d).
  
      Return types:
          * **X** (PyTorch Float Tensor) - Temporal attention scores, with shape (batch_size, num_step, num_nodes, K*d).
      """
      batch_size = X.shape[0]
      X = torch.cat((X, STE), dim=-1)
      query = self._fully_connected_q(X)
      key = self._fully_connected_k(X)
      value = self._fully_connected_v(X)
      query = torch.cat(torch.split(query, self._K, dim=-1), dim=0)
      key = torch.cat(torch.split(key, self._K, dim=-1), dim=0)
      value = torch.cat(torch.split(value, self._K, dim=-1), dim=0)
      query = query.permute(0, 2, 1, 3)
      key = key.permute(0, 2, 3, 1)
      value = value.permute(0, 2, 1, 3)
      attention = torch.matmul(query, key)
      attention /= self._d**0.5
      if self._mask:
          batch_size = X.shape[0]
          num_step = X.shape[1]
          num_nodes = X.shape[2]
          mask = torch.ones(num_step, num_step).to(X.device)
          mask = torch.tril(mask)
          mask = torch.unsqueeze(torch.unsqueeze(mask, dim=0), dim=0)
          mask = mask.repeat(self._K * batch_size, num_nodes, 1, 1)
          mask = mask.to(torch.bool)
          condition = torch.FloatTensor([-(2**15) + 1]).to(X.device)
          attention = torch.where(mask, attention, condition)
      attention = F.softmax(attention, dim=-1)
      X = torch.matmul(attention, value)
      X = X.permute(0, 2, 1, 3)
      X = torch.cat(torch.split(X, batch_size, dim=0), dim=-1)
      X = self._fully_connected(X)
      del query, key, value, attention
      return X
  
imports: import math
  from typing import Union, Callable, Optional
  import torch
  import torch.nn
  import torch.nn.functional
label_desc: Use image node_id 2 for calling the TemporalAttention obj's underlying member method code with example usage: obj.forward(X, STE) and returns: X