code: def __init__(self, config, use_bias=False):
      super().__init__()
      self.num_attention_heads = config.num_attention_heads
      self.hidden_size = config.hidden_size
      self.head_size = self.hidden_size // self.num_attention_heads
  
      self.rotary_ndims = int(self.head_size * config.rotary_pct)
      self.rotary_emb = RotaryEmbedding(
          self.rotary_ndims,
          config.max_position_embeddings,
          base=config.rotary_emb_base,
      )
      self.max_positions = config.max_position_embeddings
      self.attention_dropout = nn.Dropout(config.attention_dropout)
      self.norm_factor = torch.sqrt(
          torch.tensor(self.head_size, dtype=torch.float32)
      ).to(torch.get_default_dtype())
  
      self.query_key_value = nn.Linear(
          config.hidden_size, 3 * config.hidden_size, bias=False
      )
      self.dense = nn.Linear(
          config.hidden_size, config.hidden_size, bias=False
      )
      # Activate bias if the last layer
      self.use_bias = use_bias
      self.dense_bias = (
          nn.Parameter(torch.zeros(config.hidden_size))
          if use_bias
          else None
      )
  
imports: from typing import Optional, Tuple, Union
  import torch
  import torch.utils.checkpoint
  from torch import Tensor, nn
  from torch.nn import CrossEntropyLoss
  from ...activations import ACT2FN
  from ...file_utils import add_start_docstrings, add_start_docstrings_to_model_forward, replace_return_docstrings
  from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
  from ...modeling_utils import PreTrainedModel
  from ...utils import logging
  from .configuration_gpt_neox_japanese import GPTNeoXJapaneseConfig
label_desc: Use image node_id 1 to create a new GPTNeoXJapaneseAttention object from inherited base classes: nn with example: obj = GPTNeoXJapaneseAttention(config, use_bias)