code: def test_batch_normalization_train():
      utt.seed_rng()
  
      for axes in ("per-activation", "spatial", (1, 2, 3, 4)):
          for vartype in (T.tensor5, T.tensor3, T.vector):
              x, scale, bias, running_mean, running_var = (
                  vartype(n)
                  for n in (
                      "x",
                      "scale",
                      "bias",
                      "running_mean",
                      "running_var",
                  )
              )
              ndim = x.ndim
              eps = 5e-3  # some non-standard value to test if it's used
              running_average_factor = 0.3
  
              # remove non-existing axes
              if isinstance(axes, tuple):
                  axes = tuple(i for i in axes if i < ndim)
              if len(axes) == 0:
                  continue
  
              # forward pass
              (
                  out,
                  x_mean,
                  x_invstd,
                  out_running_mean,
                  out_running_var,
              ) = bn.batch_normalization_train(
                  x,
                  scale,
                  bias,
                  axes,
                  eps,
                  running_average_factor,
                  running_mean,
                  running_var,
              )
              # reference forward pass
              if axes == "per-activation":
                  axes2 = (0,)
              elif axes == "spatial":
                  axes2 = (0,) + tuple(range(2, ndim))
              else:
                  axes2 = axes
              x_mean2 = x.mean(axis=axes2, keepdims=True)
              x_var2 = x.var(axis=axes2, keepdims=True)
              x_invstd2 = T.inv(T.sqrt(x_var2 + eps))
              scale2 = T.addbroadcast(scale, *axes2)
              bias2 = T.addbroadcast(bias, *axes2)
              out2 = (x - x_mean2) * (scale2 * x_invstd2) + bias2
              m = T.cast(
                  T.prod(x.shape) / T.prod(scale.shape),
                  theano.config.floatX,
              )
              out_running_mean2 = (
                  running_mean * (1 - running_average_factor)
                  + x_mean2 * running_average_factor
              )
              out_running_var2 = (
                  running_var * (1 - running_average_factor)
                  + (m / (m - 1)) * x_var2 * running_average_factor
              )
              # backward pass
              dy = vartype("dy")
              grads = T.grad(
                  None, wrt=[x, scale, bias], known_grads={out: dy}
              )
              # reference backward pass
              grads2 = T.grad(
                  None, wrt=[x, scale, bias], known_grads={out2: dy}
              )
              # second-order backward pass
              dx = vartype("dinputs")
              dscale = vartype("dscale")
              dbias = vartype("dbias")
              grad_grads = T.grad(
                  None,
                  wrt=[x, dy, scale],
                  known_grads=OrderedDict(
                      {grads[0]: dx, grads[1]: dscale, grads[2]: dbias}
                  ),
                  consider_constant=[
                      x,
                      dy,
                      scale,
                      bias,
                      x_mean,
                      x_invstd,
                      running_mean,
                      running_var,
                  ],
                  return_disconnected="zero",
              )
              # reference second-order backward pass
              grad_grads2 = T.grad(
                  None,
                  wrt=[x, dy, scale],
                  known_grads=OrderedDict(
                      {
                          grads2[0]: dx,
                          grads2[1]: dscale,
                          grads2[2]: dbias,
                      }
                  ),
                  consider_constant=[
                      x,
                      dy,
                      scale,
                      bias,
                      x_mean2,
                      x_var2,
                      running_mean,
                      running_var,
                  ],
                  return_disconnected="zero",
              )
              # compile
              f = theano.function(
                  [
                      x,
                      scale,
                      bias,
                      running_mean,
                      running_var,
                      dy,
                      dx,
                      dscale,
                      dbias,
                  ],
                  [
                      out,
                      x_mean,
                      x_invstd,
                      out_running_mean,
                      out_running_var,
                      out2,
                      x_mean2,
                      x_invstd2,
                      out_running_mean2,
                      out_running_var2,
                  ]
                  + grads
                  + grads2
                  + grad_grads
                  + grad_grads2,
              )
              # check if the abstract Ops have been replaced
              assert not any(
                  [
                      isinstance(
                          n.op,
                          (
                              bn.AbstractBatchNormTrain,
                              bn.AbstractBatchNormInference,
                              bn.AbstractBatchNormTrainGrad,
                          ),
                      )
                      for n in f.maker.fgraph.toposort()
                  ]
              )
              # run
              for data_shape in (
                  (5, 10, 30, 40, 10),
                  (4, 3, 1, 1, 1),
                  (2, 3, 5, 5, 5),
              ):
                  data_shape = data_shape[:ndim]
                  param_shape = tuple(
                      1 if d in axes2 else s
                      for d, s in enumerate(data_shape)
                  )
                  X = 4 + 3 * np.random.randn(*data_shape).astype(
                      theano.config.floatX
                  )
                  Dy = -1 + 2 * np.random.randn(*data_shape).astype(
                      theano.config.floatX
                  )
                  Scale = np.random.randn(*param_shape).astype(
                      theano.config.floatX
                  )
                  Bias = np.random.randn(*param_shape).astype(
                      theano.config.floatX
                  )
                  Running_mean = np.random.randn(*param_shape).astype(
                      theano.config.floatX
                  )
                  Running_var = np.random.randn(*param_shape).astype(
                      theano.config.floatX
                  )
                  Dx = 4 + 3 * np.random.randn(*data_shape).astype(
                      theano.config.floatX
                  )
                  Dscale = -1 + 2 * np.random.randn(
                      *param_shape
                  ).astype(theano.config.floatX)
                  Dbias = np.random.randn(*param_shape).astype(
                      theano.config.floatX
                  )
  
                  outputs = f(
                      X,
                      Scale,
                      Bias,
                      Running_mean,
                      Running_var,
                      Dy,
                      Dx,
                      Dscale,
                      Dbias,
                  )
                  # compare outputs
                  utt.assert_allclose(outputs[0], outputs[0 + 5])  # out
                  utt.assert_allclose(
                      outputs[1], outputs[1 + 5]
                  )  # mean
                  utt.assert_allclose(
                      outputs[2], outputs[2 + 5]
                  )  # invstd
                  utt.assert_allclose(
                      outputs[3], outputs[3 + 5]
                  )  # running_mean
                  utt.assert_allclose(
                      np.nan_to_num(outputs[4]),
                      np.nan_to_num(outputs[4 + 5]),
                  )  # running_var
                  # compare gradients
                  utt.assert_allclose(
                      outputs[10], outputs[10 + 3], atol=1e-4
                  )  # dx
                  utt.assert_allclose(
                      outputs[11], outputs[11 + 3], rtol=2e-4, atol=1e-4
                  )  # dscale
                  utt.assert_allclose(
                      outputs[12], outputs[12 + 3]
                  )  # dbias
                  # compare second-order gradients
                  utt.assert_allclose(
                      outputs[16], outputs[16 + 3], atol=1e-4
                  )  # ddx
                  utt.assert_allclose(
                      outputs[17], outputs[17 + 3]
                  )  # ddy
                  utt.assert_allclose(
                      outputs[18], outputs[18 + 3], rtol=3e-4, atol=1e-4
                  )
  
file_path: Theano/theano/tensor/nnet/tests/test_bn.py
imports: from __future__ import absolute_import, print_function, division
  import theano
  import theano.tensor
  from theano.tests import unittest_tools
  import numpy
  from collections import OrderedDict
  from theano.tensor.nnet import bn
label_desc: Use image node_id 4 for calling a global function with example usage: test_batch_normalization_train() without return types