class_docstr: ## Estimated Batch Normalization
  
  When input $X \in \mathbb{R}^{B     imes C  imes H  imes W}$ is a batch of image representations,
  where $B$ is the batch size, $C$ is the number of channels, $H$ is the height and $W$ is the width.
  $\gamma \in \mathbb{R}^{C}$ and $eta \in \mathbb{R}^{C}$.
  
  $$\dot{X}_{\cdot, C, \cdot, \cdot} = \gamma_C
  rac{X_{\cdot, C, \cdot, \cdot} - \hat{\mu}_C}{\hat{\sigma}_C}
  + eta_C$$
  
  where,
  
  egin{align}
  \hat{\mu}_C &\longleftarrow (1 - r)\hat{\mu}_C + r rac{1}{B H W} \sum_{b,h,w} X_{b,c,h,w} \
  \hat{\sigma}^2_C &\longleftarrow (1 - r)\hat{\sigma}^2_C + r rac{1}{B H W} \sum_{b,h,w} ig(X_{b,c,h,w} - \hat{\mu}_C ig)^2
  \end{align}
  
  are the running mean and variances. $r$ is the momentum for calculating the exponential mean.
class_docstr_tok: ["#","#","Estimated","Batch","Normalization","When","input","$","X","\\in","\\mathbb","{","R","}","^","{","B","imes","C","imes","H","imes","W","}","$","is","a","batch","of","image","representations",",","where","$","B","$","is","the","batch","size",",","$","C","$","is","the","number","of","channels",",","$","H","$","is","the","height","and","$","W","$","is","the","width",".","$","\\gamma","\\in","\\mathbb","{","R","}","^","{","C","}","$","and","$","\beta","\\in","\\mathbb","{","R","}","^","{","C","}","$",".","$","$","\\dot","{","X","}","_","{","\\cdot",",","C",",","\\cdot",",","\\cdot","}","=","\\gamma_C","rac","{","X_","{","\\cdot",",","C",",","\\cdot",",","\\cdot","}","-","\\hat","{","\\mu","}","_C","}","{","\\hat","{","\\sigma","}","_C","}","+","\beta_C","$","$","where",",","\begin","{","align","}","\\hat","{","\\mu","}","_C","&","\\longleftarrow","(","1","-","r",")","\\hat","{","\\mu","}","_C","+","r","rac","{","1","}","{","B","H","W","}","\\sum_","{","b",",","h",",","w","}","X_","{","b",",","c",",","h",",","w","}","\\","\\hat","{","\\sigma","}","^2_C","&","\\longleftarrow","(","1","-","r",")","\\hat","{","\\sigma","}","^2_C","+","r","rac","{","1","}","{","B","H","W","}","\\sum_","{","b",",","h",",","w","}","\big","(","X_","{","b",",","c",",","h",",","w","}","-","\\hat","{","\\mu","}","_C","\big",")","^2","\\end","{","align","}","are","the","running","mean","and","variances",".","$","r","$","is","the","momentum","for","calculating","the","exponential","mean","."]
docstr: * `channels` is the number of features in the input
  * `eps` is $\epsilon$, used in $\sqrt{Var[x^{(k)}] + \epsilon}$ for numerical stability
  * `momentum` is the momentum in taking the exponential moving average
  * `estimate` is whether to use running mean and variance for batch norm
docstr_tok: ["*","`","channels","`","is","the","number","of","features","in","the","input","*","`","eps","`","is","$","\\epsilon","$",",","used","in","$","\\sqrt","{","Var","[","x^","{","(","k",")","}","]","+","\\epsilon","}","$","for","numerical","stability","*","`","momentum","`","is","the","momentum","in","taking","the","exponential","moving","average","*","`","estimate","`","is","whether","to","use","running","mean","and","variance","for","batch","norm"]
code: def __init__(
      self,
      channels: int,
      eps: float = 1e-5,
      momentum: float = 0.1,
      affine: bool = True,
  ):
      """
      * `channels` is the number of features in the input
      * `eps` is $\epsilon$, used in $\sqrt{Var[x^{(k)}] + \epsilon}$ for numerical stability
      * `momentum` is the momentum in taking the exponential moving average
      * `estimate` is whether to use running mean and variance for batch norm
      """
      super().__init__()
  
      self.eps = eps
      self.momentum = momentum
      self.affine = affine
      self.channels = channels
  
      # Channel wise transformation parameters
      if self.affine:
          self.scale = nn.Parameter(torch.ones(channels))
          self.shift = nn.Parameter(torch.zeros(channels))
  
      # Tensors for $\hat{\mu}_C$ and $\hat{\sigma}^2_C$
      self.register_buffer("exp_mean", torch.zeros(channels))
      self.register_buffer("exp_var", torch.ones(channels))
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/normalization/batch_channel_norm/__init__.py
imports: import torch
  from torch import nn
  from labml_helpers.module import Module
  from labml_nn.normalization.batch_norm import BatchNorm
label_desc: Use image node_id 1 to create a new EstimatedBatchNorm object from inherited base classes: Module with example: obj = EstimatedBatchNorm(channels, eps, momentum, affine)