class_docstr: The Transformer Encoder Layer in Mobile Bert
class_docstr_tok: ["The","Transformer","Encoder","Layer","in","Mobile","Bert"]
docstr: Parameters
  ----------
  use_bottleneck
      Whether to use the bottleneck layer.
  units
      size of inter-bottleneck
  real_units
      size of intra-bottleneck
  hidden_size
      size of feed-forward network
  num_heads
  num_stacked_ffn
  attention_dropout_prob
  hidden_dropout_prob
  activation_dropout_prob
  activation
  normalization
  layer_norm_eps
      onlyv valid when normalization is 'layer_norm'
  use_qkv_bias
  weight_initializer
  bias_initializer
  dtype
      Data type of the block
  layout
      Layout of the input + output
docstr_tok: ["Parameters","--","--","--","--","--","use_bottleneck","Whether","to","use","the","bottleneck","layer",".","units","size","of","inter-bottleneck","real_units","size","of","intra-bottleneck","hidden_size","size","of","feed-forward","network","num_heads","num_stacked_ffn","attention_dropout_prob","hidden_dropout_prob","activation_dropout_prob","activation","normalization","layer_norm_eps","onlyv","valid","when","normalization","is","'layer_norm'","use_qkv_bias","weight_initializer","bias_initializer","dtype","Data","type","of","the","block","layout","Layout","of","the","input","+","output"]
code: def __init__(
      self,
      use_bottleneck: bool = True,
      units: int = 512,
      real_units: int = 128,
      hidden_size: int = 2048,
      num_heads: int = 8,
      num_stacked_ffn: int = 1,
      bottleneck_strategy: str = "qk_sharing",
      attention_dropout_prob: float = 0.1,
      hidden_dropout_prob: float = 0.1,
      activation_dropout_prob: float = 0.0,
      activation: str = "gelu",
      normalization: str = "layer_norm",
      layer_norm_eps: float = 1e-12,
      use_qkv_bias: bool = True,
      weight_initializer: Optional[InitializerType] = None,
      bias_initializer: Optional[InitializerType] = "zeros",
      dtype="float32",
      layout="NT",
  ):
      """
  
      Parameters
      ----------
      use_bottleneck
          Whether to use the bottleneck layer.
      units
          size of inter-bottleneck
      real_units
          size of intra-bottleneck
      hidden_size
          size of feed-forward network
      num_heads
      num_stacked_ffn
      attention_dropout_prob
      hidden_dropout_prob
      activation_dropout_prob
      activation
      normalization
      layer_norm_eps
          onlyv valid when normalization is 'layer_norm'
      use_qkv_bias
      weight_initializer
      bias_initializer
      dtype
          Data type of the block
      layout
          Layout of the input + output
      """
      super().__init__()
      self._use_bottleneck = use_bottleneck
      self._units = units
      self._real_units = real_units
      self._num_heads = num_heads
      self._num_stacked_ffn = num_stacked_ffn
      self._bottleneck_strategy = bottleneck_strategy
      self._dtype = dtype
      self._layout = layout
      assert (
          real_units % num_heads == 0
      ), "units must be divisive by the number of heads"
      self.dropout_layer = nn.Dropout(hidden_dropout_prob)
      if use_bottleneck:
          self.in_bottleneck_proj = nn.Dense(
              units=real_units,
              in_units=units,
              flatten=False,
              weight_initializer=weight_initializer,
              bias_initializer=bias_initializer,
              dtype=self._dtype,
          )
          self.in_bottleneck_ln = get_norm_layer(
              normalization=normalization,
              in_channels=real_units,
              epsilon=layer_norm_eps,
          )
          self.out_bottleneck_proj = nn.Dense(
              units=units,
              in_units=real_units,
              flatten=False,
              weight_initializer=weight_initializer,
              bias_initializer=bias_initializer,
              dtype=self._dtype,
          )
          self.out_bottleneck_ln = get_norm_layer(
              normalization=normalization,
              in_channels=units,
              epsilon=layer_norm_eps,
          )
  
          if bottleneck_strategy == "qk_sharing":
              self.shared_qk = nn.Dense(
                  units=real_units,
                  in_units=units,
                  flatten=False,
                  weight_initializer=weight_initializer,
                  bias_initializer=bias_initializer,
                  dtype=self._dtype,
              )
              self.shared_qk_ln = get_norm_layer(
                  normalization=normalization,
                  in_channels=real_units,
                  epsilon=layer_norm_eps,
              )
      self.attention_proj = nn.Dense(
          units=real_units,
          flatten=False,
          in_units=real_units,
          use_bias=True,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      # The in_units of qkv varies according to the sharing strategy
      if self._use_bottleneck:
          if self._bottleneck_strategy == "qk_sharing":
              attn_query_in_units = real_units
              attn_key_in_units = real_units
              attn_value_in_units = units
          elif self._bottleneck_strategy == "from_bottleneck":
              attn_query_in_units = real_units
              attn_key_in_units = real_units
              attn_value_in_units = real_units
          elif self._bottleneck_strategy == "from_input":
              attn_query_in_units = units
              attn_key_in_units = units
              attn_value_in_units = units
          else:
              raise NotImplementedError
      else:
          attn_query_in_units = units
          attn_key_in_units = units
          attn_value_in_units = units
      self.attn_query = nn.Dense(
          units=real_units,
          in_units=attn_query_in_units,
          flatten=False,
          use_bias=use_qkv_bias,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      self.attn_key = nn.Dense(
          units=real_units,
          in_units=attn_key_in_units,
          flatten=False,
          use_bias=use_qkv_bias,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      self.attn_value = nn.Dense(
          units=real_units,
          in_units=attn_value_in_units,
          flatten=False,
          use_bias=use_qkv_bias,
          weight_initializer=weight_initializer,
          bias_initializer=bias_initializer,
          dtype=self._dtype,
      )
      attention_layout = "NTK" if self._layout == "NT" else "TNK"
      self.attention_cell = MultiHeadAttentionCell(
          query_units=real_units,
          num_heads=num_heads,
          attention_dropout=attention_dropout_prob,
          scaled=True,
          dtype=self._dtype,
          layout=attention_layout,
      )
      self.layer_norm = get_norm_layer(
          normalization=normalization,
          in_channels=real_units,
          epsilon=layer_norm_eps,
      )
  
      self.stacked_ffn = nn.HybridSequential()
      for ffn_idx in range(num_stacked_ffn):
          is_last_ffn = ffn_idx == (num_stacked_ffn - 1)
          # only apply dropout on last ffn layer if use bottleneck
          dropout = float(
              hidden_dropout_prob * (not use_bottleneck) * is_last_ffn
          )
          self.stacked_ffn.add(
              PositionwiseFFN(
                  units=real_units,
                  hidden_size=hidden_size,
                  dropout=dropout,
                  activation_dropout=activation_dropout_prob,
                  weight_initializer=weight_initializer,
                  bias_initializer=bias_initializer,
                  activation=activation,
                  normalization=normalization,
                  layer_norm_eps=layer_norm_eps,
                  dtype=self._dtype,
              )
          )
  
imports: import os
  from typing import Tuple, Optional
  import mxnet
  from mxnet import use_np, np, npx
  from mxnet.gluon import HybridBlock, nn
  from .base import BACKBONE_REGISTRY
  from ..op import select_vectors_by_position
  from ..base import get_model_zoo_home_dir, get_repo_model_zoo_url, get_model_zoo_checksum_dir
  from ..layers import InitializerType, PositionwiseFFN, PositionalEmbedding, get_norm_layer, get_activation
  from ..initializer import TruncNorm
  from ..utils.config import CfgNode
  from ..utils.misc import load_checksum_stats, download
  from ..utils.registry import Registry
  from ..attention_cell import MultiHeadAttentionCell, gen_self_attn_mask
  from ..data.tokenizers import HuggingFaceWordPieceTokenizer
label_desc: Use image node_id 1 to create a new MobileBertEncoderLayer object from inherited base classes: HybridBlock with example: obj = MobileBertEncoderLayer(use_bottleneck, units, real_units, hidden_size, num_heads, num_stacked_ffn, bottleneck_strategy, attention_dropout_prob, hidden_dropout_prob, activation_dropout_prob, activation, normalization, layer_norm_eps, use_qkv_bias, weight_initializer, bias_initializer, dtype, layout)