class_docstr: Electra Model
  
  This is almost the same as bert model with embedding_size adjustable (factorized embedding).
class_docstr_tok: ["Electra","Model","This","is","almost","the","same","as","bert","model","with","embedding_size","adjustable","(","factorized","embedding",")","."]
docstr: Apply the layer-wise gradient decay
  
  .. math::
      lr = lr * layerwise_decay^(max_depth - layer_depth)
  
  Parameters
  ----------
  layerwise_decay
      Power rate of the layer-wise decay
  not_included
      A list or parameter names that not included in the layer-wise decay
  num_additional_layers
      The number of layers after the current backbone. This helps determine the max depth
docstr_tok: ["Apply","the","layer-wise","gradient","decay","..","math",":",":","lr","=","lr","*","layerwise_decay^","(","max_depth","-","layer_depth",")","Parameters","--","--","--","--","--","layerwise_decay","Power","rate","of","the","layer-wise","decay","not_included","A","list","or","parameter","names","that","not","included","in","the","layer-wise","decay","num_additional_layers","The","number","of","layers","after","the","current","backbone",".","This","helps","determine","the","max","depth"]
code: def apply_layerwise_decay(
      self,
      layerwise_decay: int,
      not_included: Optional[List[str]] = None,
      num_additional_layers: int = 2,
  ):
      """Apply the layer-wise gradient decay
  
      .. math::
          lr = lr * layerwise_decay^(max_depth - layer_depth)
  
      Parameters
      ----------
      layerwise_decay
          Power rate of the layer-wise decay
      not_included
          A list or parameter names that not included in the layer-wise decay
      num_additional_layers
          The number of layers after the current backbone. This helps determine the max depth
      """
  
      # Consider the task specific finetuning layer as the last layer, following with pooler
      # In addition, the embedding parameters have the smaller learning rate based on this
      # setting.
      max_depth = self.num_layers + num_additional_layers
      for _, value in self.collect_params(".*embed*").items():
          value.lr_mult = layerwise_decay**max_depth
  
      for layer_depth, layer in enumerate(
          self.encoder.all_encoder_layers
      ):
          layer_params = layer.collect_params()
          for key, value in layer_params.items():
              if not_included:
                  for pn in not_included:
                      if pn in key:
                          continue
              value.lr_mult = layerwise_decay ** (
                  max_depth - (layer_depth + 1)
              )
  
imports: import os
  from typing import Tuple, Optional, List
  import mxnet
  from mxnet import use_np, np, npx
  from mxnet.gluon import HybridBlock, nn
  from .base import BACKBONE_REGISTRY
  from .transformer import TransformerEncoderLayer
  from ..op import gumbel_softmax, select_vectors_by_position, add_vectors_by_position, update_vectors_by_position
  from ..base import get_model_zoo_home_dir, get_repo_model_zoo_url, get_model_zoo_checksum_dir
  from ..layers import PositionalEmbedding, get_activation
  from ..initializer import TruncNorm
  from ..utils.config import CfgNode
  from ..utils.misc import load_checksum_stats, download
  from ..utils.registry import Registry
  from ..attention_cell import gen_self_attn_mask
  from ..data.tokenizers import HuggingFaceWordPieceTokenizer
label_desc: Use image node_id 5 for calling the ElectraModel obj's underlying member method code with example usage: obj.apply_layerwise_decay(layerwise_decay, not_included, num_additional_layers) without return types