class_docstr: Implementation of an AdapterFusion block.
code: def __init__(
      self,
      config: AdapterFusionConfig,
      dense_size,
      attention_probs_dropout_prob,
  ):
      super(BertFusion, self).__init__()
      # if config.hidden_size % config.num_attention_heads != 0:
      #     raise ValueError(
      #         "The hidden size (%d) is not a multiple of the number of attention "
      #         "heads (%d)" % (config.hidden_size, config.num_attention_heads))
      self.config = config
  
      self.dense_size = dense_size
      self.dropout = nn.Dropout(attention_probs_dropout_prob)
  
      if (
          not self.config["query"]
          and not self.config["key"]
          and not self.config["value"]
      ):
          self.dense = nn.Linear(self.dense_size, 1)
  
      if self.config["query"]:
          self.query = nn.Linear(self.dense_size, self.dense_size)
          self.query.apply(Adapter.init_bert_weights)
  
      if self.config["key"]:
          self.key = nn.Linear(self.dense_size, self.dense_size)
          self.key.apply(Adapter.init_bert_weights)
  
      if self.config["value"]:
          self.value = nn.Linear(
              self.dense_size, self.dense_size, bias=False
          )
          self.value.apply(Adapter.init_bert_weights)
          if self.config["value_initialized"]:
              self.value.weight.data = (
                  torch.zeros(self.dense_size, self.dense_size)
                  + 0.000001
              ).fill_diagonal_(1.0)
  
      if self.config["temperature"]:
          self.T = 50.0
      else:
          self.T = 1.0
      self.reduction = self.T / 1000.0
  
imports: import math
  import torch
  from torch import nn
  from transformers.activations import get_activation
  from ..configuration import AdapterFusionConfig, BnConfig
  from ..context import ForwardContext
label_desc: Use image node_id 1 to create a new BertFusion object from inherited base classes: nn with example: obj = BertFusion(config, dense_size, attention_probs_dropout_prob)