class_docstr: TransformerEncoderLayer is based on "Attention
  is all you Need" (https://arxiv.org/pdf/1706.03762.pdf).
class_docstr_tok: ["TransformerEncoderLayer","is","based","on","``","Attention","is","all","you","Need","''","(","https",":","\/\/arxiv.org\/pdf\/1706.03762.pdf",")","."]
code: def __init__(self, config, **kwargs):
      super().__init__(**kwargs)
      model_config = config["model"]["net"]["structure"]
      self.head_num = model_config.get("head_num")
      self.hidden_dim = model_config.get("hidden_dim")
      self.inner_size = model_config.get("inner_size")
      self.feed_forward_act = config.get("feed_forward_act", "relu")
      self.dropout_rate = config.get("dropout_rate", 0.0)
  
      self.self_attn_layer = delta.layers.MultiHeadAttention(
          self.hidden_dim, self.head_num
      )
      self.feed_forward_layer = delta.layers.PositionwiseFeedForward(
          self.hidden_dim, self.inner_size, self.feed_forward_act
      )
      self.embed_dense = tf.keras.layers.Dense(self.hidden_dim)
  
      self.self_attn_dropout = tf.keras.layers.Dropout(
          self.dropout_rate
      )
      self.feed_forward_dropout = tf.keras.layers.Dropout(
          self.dropout_rate
      )
  
      self.self_attn_norm = tf.keras.layers.LayerNormalization(
          epsilon=1e-6
      )
      self.feed_forward_norm = tf.keras.layers.LayerNormalization(
          epsilon=1e-6
      )
  
imports: import math
  from absl import logging
  import numpy
  import delta.compat
  from tensorflow.python.util import nest
  from delta.layers.base_layer import Layer
  from delta.layers import utils_tf
  import delta.layers
label_desc: Use image node_id 1 to create a new TransformerEncoderLayer object from inherited base classes: Layer with example: obj = TransformerEncoderLayer(config)