class_docstr: Multi-head self-attention layer with the Fixup initialization.
class_docstr_tok: ["Multi-head","self-attention","layer","with","the","Fixup","initialization","."]
code: def __init__(
      self,
      num_blocks=gin.REQUIRED,
      default_init="glorot",
      init_distribution="uniform",
      **kwargs,
  ):
      # Any arg in `kwargs` should be defined in SelfAttention constructor.
      super(SelfAttentionFixup, self).__init__(**kwargs)
      self.num_blocks = num_blocks
      self.default_init = default_init
      self.init_distribution = init_distribution
  
imports: import math
  import gin
  import mesh_tensorflow
  from mesh_tensorflow.transformer import attention
  from mesh_tensorflow.transformer import transformer
  from mesh_tensorflow.transformer import transformer_layers
  import tensorflow.compat.v1
label_desc: Use image node_id 1 to create a new SelfAttentionFixup object from inherited base classes: transformer_layers with example: obj = SelfAttentionFixup(num_blocks, default_init, init_distribution)