code: def __init__(
      self,
      dim_model: int,
      num_heads: int,
      dim_head: int,
      dtype: torch.dtype = torch.half,
      dropout_p: Optional[float] = None,
  ) -> None:
      super().__init__()
  
      self.dim_model = dim_model
      self.num_heads = num_heads
      self.dim_head = dim_head
  
      self.project_q = Linear(
          self.dim_model, self.num_heads * self.dim_head, dtype=dtype
      )
      self.project_k = Linear(
          self.dim_model, self.num_heads * self.dim_head, dtype=dtype
      )
      self.project_v = Linear(
          self.dim_model, self.num_heads * self.dim_head, dtype=dtype
      )
  
      self.attention_out = Linear(
          self.num_heads * self.dim_head, self.dim_model, dtype=dtype
      )
  
      self.softmax = torch.nn.Softmax(dim=-1)
  
      if dropout_p is not None:
          self.dropout = torch.nn.Dropout(p=dropout_p)
      else:
          self.dropout = None
  
file_path: DeepKE/example/llm/CPM-Bee/src/cpm_live/native_layers/attention.py
imports: from typing import Optional, Tuple
  import torch
  import math
  from .linear import Linear
label_desc: Use image node_id 1 to create a new Attention object from inherited base classes: torch.nn with example: obj = Attention(dim_model, num_heads, dim_head, dtype, dropout_p)