class_docstr: lasagne.layers.BatchNormLayer(incoming, axes='auto', epsilon=1e-4,
  alpha=0.1, beta=lasagne.init.Constant(0), gamma=lasagne.init.Constant(1),
  mean=lasagne.init.Constant(0), inv_std=lasagne.init.Constant(1), **kwargs)
  
  Batch Normalization
  
  This layer implements batch normalization of its inputs, following [1]_:
  
  .. math::
      y = \frac{x - \mu}{\sqrt{\sigma^2 + \epsilon}} \gamma + \beta
  
  That is, the input is normalized to zero mean and unit variance, and then
  linearly transformed. The crucial part is that the mean and variance are
  computed across the batch dimension, i.e., over examples, not per example.
  
  During training, :math:`\mu` and :math:`\sigma^2` are defined to be the
  mean and variance of the current input mini-batch :math:`x`, and during
  testing, they are replaced with average statistics over the training
  data. Consequently, this layer has four stored parameters: :math:`\beta`,
  :math:`\gamma`, and the averages :math:`\mu` and :math:`\sigma^2`
  (nota bene: instead of :math:`\sigma^2`, the layer actually stores
  :math:`1 / \sqrt{\sigma^2 + \epsilon}`, for compatibility to cuDNN).
  By default, this layer learns the average statistics as exponential moving
  averages computed during training, so it can be plugged into an existing
  network without any changes of the training procedure (see Notes).
  
  Parameters
  ----------
  incoming : a :class:`Layer` instance or a tuple
      The layer feeding into this layer, or the expected input shape
  axes : 'auto', int or tuple of int
      The axis or axes to normalize over. If ``'auto'`` (the default),
      normalize over all axes except for the second: this will normalize over
      the minibatch dimension for dense layers, and additionally over all
      spatial dimensions for convolutional layers.
  epsilon : scalar
      Small constant :math:`\epsilon` added to the variance before taking
      the square root and dividing by it, to avoid numerical problems
  alpha : scalar
      Coefficient for the exponential moving average of batch-wise means and
      standard deviations computed during training; the closer to one, the
      more it will depend on the last batches seen
  beta : Theano shared variable, expression, numpy array, callable or None
      Initial value, expression or initializer for :math:`\beta`. Must match
      the incoming shape, skipping all axes in `axes`. Set to ``None`` to fix
      it to 0.0 instead of learning it.
      See :func:`lasagne.utils.create_param` for more information.
  gamma : Theano shared variable, expression, numpy array, callable or None
      Initial value, expression or initializer for :math:`\gamma`. Must
      match the incoming shape, skipping all axes in `axes`. Set to ``None``
      to fix it to 1.0 instead of learning it.
      See :func:`lasagne.utils.create_param` for more information.
  mean : Theano shared variable, expression, numpy array, or callable
      Initial value, expression or initializer for :math:`\mu`. Must match
      the incoming shape, skipping all axes in `axes`.
      See :func:`lasagne.utils.create_param` for more information.
  inv_std : Theano shared variable, expression, numpy array, or callable
      Initial value, expression or initializer for :math:`1 / \sqrt{
      \sigma^2 + \epsilon}`. Must match the incoming shape, skipping all
      axes in `axes`.
      See :func:`lasagne.utils.create_param` for more information.
  **kwargs
      Any additional keyword arguments are passed to the :class:`Layer`
      superclass.
  
  Notes
  -----
  This layer should be inserted between a linear transformation (such as a
  :class:`DenseLayer`, or :class:`Conv2DLayer`) and its nonlinearity. The
  convenience function :func:`batch_norm` modifies an existing layer to
  insert batch normalization in front of its nonlinearity.
  
  The behavior can be controlled by passing keyword arguments to
  :func:`lasagne.layers.get_output()` when building the output expression
  of any network containing this layer.
  
  During training, [1]_ normalize each input mini-batch by its statistics
  and update an exponential moving average of the statistics to be used for
  validation. This can be achieved by passing ``deterministic=False``.
  For validation, [1]_ normalize each input mini-batch by the stored
  statistics. This can be achieved by passing ``deterministic=True``.
  
  For more fine-grained control, ``batch_norm_update_averages`` can be passed
  to update the exponential moving averages (``True``) or not (``False``),
  and ``batch_norm_use_averages`` can be passed to use the exponential moving
  averages for normalization (``True``) or normalize each mini-batch by its
  own statistics (``False``). These settings override ``deterministic``.
  
  Note that for testing a model after training, [1]_ replace the stored
  exponential moving average statistics by fixing all network weights and
  re-computing average statistics over the training data in a layerwise
  fashion. This is not part of the layer implementation.
  
  In case you set `axes` to not include the batch dimension (the first axis,
  usually), normalization is done per example, not across examples. This does
  not require any averages, so you can pass ``batch_norm_update_averages``
  and ``batch_norm_use_averages`` as ``False`` in this case.
  
  See also
  --------
  batch_norm : Convenience function to apply batch normalization to a layer
  
  References
  ----------
  .. [1] Ioffe, Sergey and Szegedy, Christian (2015):
         Batch Normalization: Accelerating Deep Network Training by Reducing
         Internal Covariate Shift. http://arxiv.org/abs/1502.03167.
class_docstr_tok: ["lasagne.layers.BatchNormLayer","(","incoming",",","axes='auto","'",",","epsilon=1e-4",",","alpha=0.1",",","beta=lasagne.init.Constant","(","0",")",",","gamma=lasagne.init.Constant","(","1",")",",","mean=lasagne.init.Constant","(","0",")",",","inv_std=lasagne.init.Constant","(","1",")",",","*","*","kwargs",")","Batch","Normalization","This","layer","implements","batch","normalization","of","its","inputs",",","following","[","1","]","_",":","..","math",":",":","y","=","\\frac","{","x","-","\\mu","}","{","\\sqrt","{","\\sigma^2","+","\\epsilon","}","}","\\gamma","+","\\beta","That","is",",","the","input","is","normalized","to","zero","mean","and","unit","variance",",","and","then","linearly","transformed",".","The","crucial","part","is","that","the","mean","and","variance","are","computed","across","the","batch","dimension",",","i.e.",",","over","examples",",","not","per","example",".","During","training",",",":","math",":","`","\\mu","`","and",":","math",":","`","\\sigma^2","`","are","defined","to","be","the","mean","and","variance","of","the","current","input","mini-batch",":","math",":","`","x","`",",","and","during","testing",",","they","are","replaced","with","average","statistics","over","the","training","data",".","Consequently",",","this","layer","has","four","stored","parameters",":",":","math",":","`","\\beta","`",",",":","math",":","`","\\gamma","`",",","and","the","averages",":","math",":","`","\\mu","`","and",":","math",":","`","\\sigma^2","`","(","nota","bene",":","instead","of",":","math",":","`","\\sigma^2","`",",","the","layer","actually","stores",":","math",":","`","1","\/","\\sqrt","{","\\sigma^2","+","\\epsilon","}","`",",","for","compatibility","to","cuDNN",")",".","By","default",",","this","layer","learns","the","average","statistics","as","exponential","moving","averages","computed","during","training",",","so","it","can","be","plugged","into","an","existing","network","without","any","changes","of","the","training","procedure","(","see","Notes",")",".","Parameters","--","--","--","--","--","incoming",":","a",":","class",":","`","Layer","`","instance","or","a","tuple","The","layer","feeding","into","this","layer",",","or","the","expected","input","shape","axes",":","'auto","'",",","int","or","tuple","of","int","The","axis","or","axes","to","normalize","over",".","If","``","'auto","'","``","(","the","default",")",",","normalize","over","all","axes","except","for","the","second",":","this","will","normalize","over","the","minibatch","dimension","for","dense","layers",",","and","additionally","over","all","spatial","dimensions","for","convolutional","layers",".","epsilon",":","scalar","Small","constant",":","math",":","`","\\epsilon","`","added","to","the","variance","before","taking","the","square","root","and","dividing","by","it",",","to","avoid","numerical","problems","alpha",":","scalar","Coefficient","for","the","exponential","moving","average","of","batch-wise","means","and","standard","deviations","computed","during","training",";","the","closer","to","one",",","the","more","it","will","depend","on","the","last","batches","seen","beta",":","Theano","shared","variable",",","expression",",","numpy","array",",","callable","or","None","Initial","value",",","expression","or","initializer","for",":","math",":","`","\\beta","`",".","Must","match","the","incoming","shape",",","skipping","all","axes","in","`","axes","`",".","Set","to","``","None","``","to","fix","it","to","0.0","instead","of","learning","it",".","See",":","func",":","`","lasagne.utils.create_param","`","for","more","information",".","gamma",":","Theano","shared","variable",",","expression",",","numpy","array",",","callable","or","None","Initial","value",",","expression","or","initializer","for",":","math",":","`","\\gamma","`",".","Must","match","the","incoming","shape",",","skipping","all","axes","in","`","axes","`",".","Set","to","``","None","``","to","fix","it","to","1.0","instead","of","learning","it",".","See",":","func",":","`","lasagne.utils.create_param","`","for","more","information",".","mean",":","Theano","shared","variable",",","expression",",","numpy","array",",","or","callable","Initial","value",",","expression","or","initializer","for",":","math",":","`","\\mu","`",".","Must","match","the","incoming","shape",",","skipping","all","axes","in","`","axes","`",".","See",":","func",":","`","lasagne.utils.create_param","`","for","more","information",".","inv_std",":","Theano","shared","variable",",","expression",",","numpy","array",",","or","callable","Initial","value",",","expression","or","initializer","for",":","math",":","`","1","\/","\\sqrt","{","\\sigma^2","+","\\epsilon","}","`",".","Must","match","the","incoming","shape",",","skipping","all","axes","in","`","axes","`",".","See",":","func",":","`","lasagne.utils.create_param","`","for","more","information",".","*","*","kwargs","Any","additional","keyword","arguments","are","passed","to","the",":","class",":","`","Layer","`","superclass",".","Notes","--","--","-","This","layer","should","be","inserted","between","a","linear","transformation","(","such","as","a",":","class",":","`","DenseLayer","`",",","or",":","class",":","`","Conv2DLayer","`",")","and","its","nonlinearity",".","The","convenience","function",":","func",":","`","batch_norm","`","modifies","an","existing","layer","to","insert","batch","normalization","in","front","of","its","nonlinearity",".","The","behavior","can","be","controlled","by","passing","keyword","arguments","to",":","func",":","`","lasagne.layers.get_output","(",")","`","when","building","the","output","expression","of","any","network","containing","this","layer",".","During","training",",","[","1","]","_","normalize","each","input","mini-batch","by","its","statistics","and","update","an","exponential","moving","average","of","the","statistics","to","be","used","for","validation",".","This","can","be","achieved","by","passing","``","deterministic=False","``",".","For","validation",",","[","1","]","_","normalize","each","input","mini-batch","by","the","stored","statistics",".","This","can","be","achieved","by","passing","``","deterministic=True","``",".","For","more","fine-grained","control",",","``","batch_norm_update_averages","``","can","be","passed","to","update","the","exponential","moving","averages","(","``","True","``",")","or","not","(","``","False","``",")",",","and","``","batch_norm_use_averages","``","can","be","passed","to","use","the","exponential","moving","averages","for","normalization","(","``","True","``",")","or","normalize","each","mini-batch","by","its","own","statistics","(","``","False","``",")",".","These","settings","override","``","deterministic","``",".","Note","that","for","testing","a","model","after","training",",","[","1","]","_","replace","the","stored","exponential","moving","average","statistics","by","fixing","all","network","weights","and","re-computing","average","statistics","over","the","training","data","in","a","layerwise","fashion",".","This","is","not","part","of","the","layer","implementation",".","In","case","you","set","`","axes","`","to","not","include","the","batch","dimension","(","the","first","axis",",","usually",")",",","normalization","is","done","per","example",",","not","across","examples",".","This","does","not","require","any","averages",",","so","you","can","pass","``","batch_norm_update_averages","``","and","``","batch_norm_use_averages","``","as","``","False","``","in","this","case",".","See","also","--","--","--","--","batch_norm",":","Convenience","function","to","apply","batch","normalization","to","a","layer","References","--","--","--","--","--","..","[","1","]","Ioffe",",","Sergey","and","Szegedy",",","Christian","(","2015",")",":","Batch","Normalization",":","Accelerating","Deep","Network","Training","by","Reducing","Internal","Covariate","Shift",".","http",":","\/\/arxiv.org\/abs\/1502.03167","."]
code: def get_output_for(
      self,
      input,
      deterministic=False,
      batch_norm_use_averages=None,
      batch_norm_update_averages=None,
      **kwargs,
  ):
      input_mean = input.mean(self.axes)
      input_inv_std = T.inv(T.sqrt(input.var(self.axes) + self.epsilon))
  
      # Decide whether to use the stored averages or mini-batch statistics
      if batch_norm_use_averages is None:
          batch_norm_use_averages = deterministic
      use_averages = batch_norm_use_averages
  
      if use_averages:
          mean = self.mean
          inv_std = self.inv_std
      else:
          mean = input_mean
          inv_std = input_inv_std
  
      # Decide whether to update the stored averages
      if batch_norm_update_averages is None:
          batch_norm_update_averages = not deterministic
      update_averages = batch_norm_update_averages
  
      if update_averages:
          # Trick: To update the stored statistics, we create memory-aliased
          # clones of the stored statistics:
          running_mean = theano.clone(self.mean, share_inputs=False)
          running_inv_std = theano.clone(
              self.inv_std, share_inputs=False
          )
          # set a default update for them:
          running_mean.default_update = (
              1 - self.alpha
          ) * running_mean + self.alpha * input_mean
          running_inv_std.default_update = (
              1 - self.alpha
          ) * running_inv_std + self.alpha * input_inv_std
          # and make sure they end up in the graph without participating in
          # the computation (this way their default_update will be collected
          # and applied, but the computation will be optimized away):
          mean += 0 * running_mean
          inv_std += 0 * running_inv_std
  
      # prepare dimshuffle pattern inserting broadcastable axes as needed
      param_axes = iter(range(input.ndim - len(self.axes)))
      pattern = [
          "x" if input_axis in self.axes else next(param_axes)
          for input_axis in range(input.ndim)
      ]
  
      # apply dimshuffle pattern to all parameters
      beta = 0 if self.beta is None else self.beta.dimshuffle(pattern)
      gamma = (
          1 if self.gamma is None else self.gamma.dimshuffle(pattern)
      )
      mean = mean.dimshuffle(pattern)
      inv_std = inv_std.dimshuffle(pattern)
  
      # normalize
      normalized = (input - mean) * (gamma * inv_std) + beta
      return normalized
  
imports: import theano
  import theano.tensor
  from ..None import init
  from ..None import nonlinearities
  from ..utils import int_types
  from .base import Layer
label_desc: Use image node_id 2 for calling the BatchNormLayer obj's underlying member method code with example usage: obj.get_output_for(input, deterministic, batch_norm_use_averages, batch_norm_update_averages) and returns: normalized