class_docstr: This class defines the pre-norm `Transformer encoder <https://arxiv.org/abs/1706.03762>`_
  Args:
      opts: Command line arguments.
      embed_dim: :math:`C_{in}` from an expected input of size :math:`(N, P, C_{in})`.
      ffn_latent_dim: Inner dimension of the FFN.
      num_heads: Number of heads in multi-head attention. Default: 8.
      attn_dropout: Dropout rate for attention in multi-head attention. Default: 0.0
      dropout: Dropout rate. Default: 0.0.
      ffn_dropout: Dropout between FFN layers. Default: 0.0.
      transformer_norm_layer: Normalization layer. Default: layer_norm.
      stochastic_dropout: Stochastic dropout setting. Default: 0.0.
  
  Shape:
      - Input: :math:`(N, P, C_{in})` where :math:`N` is batch size, :math:`P` is number of patches,
      and :math:`C_{in}` is input embedding dim
      - Output: same shape as the input
class_docstr_tok: ["This","class","defines","the","pre-norm","`","Transformer","encoder","<","https",":","\/\/arxiv.org\/abs\/1706.03762",">","`","_","Args",":","opts",":","Command","line","arguments",".","embed_dim",":",":","math",":","`","C_","{","in","}","`","from","an","expected","input","of","size",":","math",":","`","(","N",",","P",",","C_","{","in","}",")","`",".","ffn_latent_dim",":","Inner","dimension","of","the","FFN",".","num_heads",":","Number","of","heads","in","multi-head","attention",".","Default",":","8.","attn_dropout",":","Dropout","rate","for","attention","in","multi-head","attention",".","Default",":","0.0","dropout",":","Dropout","rate",".","Default",":","0.0.","ffn_dropout",":","Dropout","between","FFN","layers",".","Default",":","0.0.","transformer_norm_layer",":","Normalization","layer",".","Default",":","layer_norm",".","stochastic_dropout",":","Stochastic","dropout","setting",".","Default",":","0.0",".","Shape",":","-","Input",":",":","math",":","`","(","N",",","P",",","C_","{","in","}",")","`","where",":","math",":","`","N","`","is","batch","size",",",":","math",":","`","P","`","is","number","of","patches",",","and",":","math",":","`","C_","{","in","}","`","is","input","embedding","dim","-","Output",":","same","shape","as","the","input"]
code: def __init__(
      self,
      opts: argparse.Namespace,
      embed_dim: int,
      ffn_latent_dim: int,
      num_heads: Optional[int] = 8,
      attn_dropout: Optional[float] = 0.0,
      dropout: Optional[float] = 0.0,
      ffn_dropout: Optional[float] = 0.0,
      transformer_norm_layer: Optional[str] = "layer_norm",
      stochastic_dropout: Optional[float] = 0.0,
      *args,
      **kwargs,
  ) -> None:
      super().__init__()
  
      attn_unit = SingleHeadAttention(
          embed_dim=embed_dim, attn_dropout=attn_dropout, bias=True
      )
      if num_heads > 1:
          attn_unit = MultiHeadAttention(
              embed_dim,
              num_heads,
              attn_dropout=attn_dropout,
              bias=True,
              coreml_compatible=getattr(
                  opts, "common.enable_coreml_compatible_module", False
              ),
          )
  
      self.pre_norm_mha = nn.Sequential(
          get_normalization_layer(
              opts=opts,
              norm_type=transformer_norm_layer,
              num_features=embed_dim,
          ),
          attn_unit,
          Dropout(p=dropout),
      )
  
      act_name = build_activation_layer(opts, num_parameters=1)
      self.pre_norm_ffn = nn.Sequential(
          get_normalization_layer(
              opts=opts,
              norm_type=transformer_norm_layer,
              num_features=embed_dim,
          ),
          LinearLayer(
              in_features=embed_dim,
              out_features=ffn_latent_dim,
              bias=True,
          ),
          act_name,
          Dropout(p=ffn_dropout),
          LinearLayer(
              in_features=ffn_latent_dim,
              out_features=embed_dim,
              bias=True,
          ),
          Dropout(p=dropout),
      )
  
      self.drop_path = Identity()
      if stochastic_dropout > 0.0:
          if dropout > 0.0:
              logger.error(
                  "Stochastic dropout and dropout are mutually exclusive. "
                  "Use either of them, but not both."
                  "Got: {} and {}".format(stochastic_dropout, dropout)
              )
          self.drop_path = StochasticDepth(
              p=stochastic_dropout, mode="row"
          )
  
      self.embed_dim = embed_dim
      self.ffn_dim = ffn_latent_dim
      self.ffn_dropout = ffn_dropout
      self.stochastic_dropout = stochastic_dropout
      self.std_dropout = dropout
      self.attn_fn_name = attn_unit.__class__.__name__
      self.act_fn_name = act_name.__class__.__name__
      self.norm_type = transformer_norm_layer
  
file_path: ml-cvnets/cvnets/modules/transformer.py
imports: import argparse
  from typing import Optional
  from torch import Tensor, nn
  from cvnets.layers import ConvLayer2d, Dropout, Identity, LinearLayer, LinearSelfAttention, MultiHeadAttention, SingleHeadAttention, StochasticDepth, get_normalization_layer
  from cvnets.layers.activation import build_activation_layer
  from cvnets.modules import BaseModule
  from utils import logger
label_desc: Use image node_id 1 to create a new TransformerEncoder object from inherited base classes: BaseModule with example: obj = TransformerEncoder(opts, embed_dim, ffn_latent_dim, num_heads, attn_dropout, dropout, ffn_dropout, transformer_norm_layer, stochastic_dropout)