class_docstr: Applies Layer Normalization over a mini-batch of inputs as described in
  the paper `Layer Normalization`_ .
  
  Currently only runs on cuda() tensors.
  
  .. math::
      y = \frac{x - \mathrm{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} * \gamma + \beta
  
  The mean and standard-deviation are calculated separately over the last
  certain number dimensions which have to be of the shape specified by
  :attr:`normalized_shape`.
  :math:`\gamma` and :math:`\beta` are learnable affine transform parameters of
  :attr:`normalized_shape` if :attr:`elementwise_affine` is ``True``.
  
  .. note::
      Unlike Batch Normalization and Instance Normalization, which applies
      scalar scale and bias for each entire channel/plane with the
      :attr:`affine` option, Layer Normalization applies per-element scale and
      bias with :attr:`elementwise_affine`.
  
  This layer uses statistics computed from input data in both training and
  evaluation modes.
  
  Args:
      normalized_shape (int or list or torch.Size): input shape from an expected input
          of size
  
          .. math::
              [* \times \text{normalized}\_\text{shape}[0] \times \text{normalized}\_\text{shape}[1]
                  \times \ldots \times \text{normalized}\_\text{shape}[-1]]
  
          If a single integer is used, it is treated as a singleton list, and this module will
          normalize over the last dimension which is expected to be of that specific size.
      eps: a value added to the denominator for numerical stability. Default: 1e-5
      elementwise_affine: a boolean value that when set to ``True``, this module
          has learnable per-element affine parameters initialized to ones (for weights)
          and zeros (for biases). Default: ``True``.
  
  Shape:
      - Input: :math:`(N, *)`
      - Output: :math:`(N, *)` (same shape as input)
  
  Examples::
  
      >>> input = torch.randn(20, 5, 10, 10)
      >>> # With Learnable Parameters
      >>> m = apex.normalization.FusedLayerNorm(input.size()[1:])
      >>> # Without Learnable Parameters
      >>> m = apex.normalization.FusedLayerNorm(input.size()[1:], elementwise_affine=False)
      >>> # Normalize over last two dimensions
      >>> m = apex.normalization.FusedLayerNorm([10, 10])
      >>> # Normalize over last dimension of size 10
      >>> m = apex.normalization.FusedLayerNorm(10)
      >>> # Activating the module
      >>> output = m(input)
  
  .. _`Layer Normalization`: https://arxiv.org/abs/1607.06450
class_docstr_tok: ["Applies","Layer","Normalization","over","a","mini-batch","of","inputs","as","described","in","the","paper","`","Layer","Normalization","`","_",".","Currently","only","runs","on","cuda","(",")","tensors",".","..","math",":",":","y","=","\\frac","{","x","-","\\mathrm","{","E","}","[","x","]","}","{","\\sqrt","{","\\mathrm","{","Var","}","[","x","]","+","\\epsilon","}","}","*","\\gamma","+","\\beta","The","mean","and","standard-deviation","are","calculated","separately","over","the","last","certain","number","dimensions","which","have","to","be","of","the","shape","specified","by",":","attr",":","`","normalized_shape","`",".",":","math",":","`","\\gamma","`","and",":","math",":","`","\\beta","`","are","learnable","affine","transform","parameters","of",":","attr",":","`","normalized_shape","`","if",":","attr",":","`","elementwise_affine","`","is","``","True","``",".","..","note",":",":","Unlike","Batch","Normalization","and","Instance","Normalization",",","which","applies","scalar","scale","and","bias","for","each","entire","channel\/plane","with","the",":","attr",":","`","affine","`","option",",","Layer","Normalization","applies","per-element","scale","and","bias","with",":","attr",":","`","elementwise_affine","`",".","This","layer","uses","statistics","computed","from","input","data","in","both","training","and","evaluation","modes",".","Args",":","normalized_shape","(","int","or","list","or","torch.Size",")",":","input","shape","from","an","expected","input","of","size","..","math",":",":","[","*","\\times","\\text","{","normalized","}","\\_\\text","{","shape","}","[","0","]","\\times","\\text","{","normalized","}","\\_\\text","{","shape","}","[","1","]","\\times","\\ldots","\\times","\\text","{","normalized","}","\\_\\text","{","shape","}","[","-1","]","]","If","a","single","integer","is","used",",","it","is","treated","as","a","singleton","list",",","and","this","module","will","normalize","over","the","last","dimension","which","is","expected","to","be","of","that","specific","size",".","eps",":","a","value","added","to","the","denominator","for","numerical","stability",".","Default",":","1e-5","elementwise_affine",":","a","boolean","value","that","when","set","to","``","True","``",",","this","module","has","learnable","per-element","affine","parameters","initialized","to","ones","(","for","weights",")","and","zeros","(","for","biases",")",".","Default",":","``","True","``",".","Shape",":","-","Input",":",":","math",":","`","(","N",",","*",")","`","-","Output",":",":","math",":","`","(","N",",","*",")","`","(","same","shape","as","input",")","Examples",":",":",">",">",">","input","=","torch.randn","(","20",",","5",",","10",",","10",")",">",">",">","#","With","Learnable","Parameters",">",">",">","m","=","apex.normalization.FusedLayerNorm","(","input.size","(",")","[","1",":","]",")",">",">",">","#","Without","Learnable","Parameters",">",">",">","m","=","apex.normalization.FusedLayerNorm","(","input.size","(",")","[","1",":","]",",","elementwise_affine=False",")",">",">",">","#","Normalize","over","last","two","dimensions",">",">",">","m","=","apex.normalization.FusedLayerNorm","(","[","10",",","10","]",")",">",">",">","#","Normalize","over","last","dimension","of","size","10",">",">",">","m","=","apex.normalization.FusedLayerNorm","(","10",")",">",">",">","#","Activating","the","module",">",">",">","output","=","m","(","input",")","..","_","`","Layer","Normalization","`",":","https",":","\/\/arxiv.org\/abs\/1607.06450"]
code: def forward(self, input):
      if (
          torch.jit.is_tracing()
          or torch.jit.is_scripting()
          or not input.is_cuda
      ):
          return F.layer_norm(
              input,
              self.normalized_shape,
              self.weight,
              self.bias,
              self.eps,
          )
      if self.elementwise_affine:
          return fused_layer_norm_affine(
              input,
              self.weight,
              self.bias,
              self.normalized_shape,
              self.eps,
              self.memory_efficient,
          )
      else:
          return fused_layer_norm(
              input,
              self.normalized_shape,
              self.eps,
              self.memory_efficient,
          )
  
imports: import importlib
  import numbers
  import torch
  from torch.nn.parameter import Parameter
  from torch.nn import init
  from torch.nn import functional
  from apex._autocast_utils import _cast_if_autocast_enabled
label_desc: Use image node_id 3 for calling the FusedLayerNorm obj's underlying member method code with example usage: obj.forward(input) and returns: F, fused_layer_norm_affine, fused_layer_norm