neural_tangents/stax.py
def Dense(out_dim, W_std=1., b_std=0., W_init=_randn(1.0), b_init=_randn(1.0),
          parameterization='ntk'):
  r"""Layer constructor function for a dense (fully-connected) layer.

  Based on `jax.experimental.stax.Dense`. Has a similar API, apart from:

  W_init and b_init only change the behavior of the finite width network, and
    are not used by kernel_fn. In most cases, W_std and b_std should be used
    instead

  Args:
    parameterization: Either 'ntk' or 'standard'.
      Under ntk parameterization (https://arxiv.org/abs/1806.07572, page 3),
        weights and biases are initialized as W_ij ~ N(0,1), b_i ~ N(0,1), and
        the finite width layer equation is z_i = W_std / sqrt([width]) sum_j
        W_ij x_j + b_std b_i Under standard parameterization, weights and biases
        are initialized as W_ij ~ N(0,W_std^2/[width]), b_i ~ N(0,b_std^2), and
        the finite width layer equation is z_i = \sum_j W_ij x_j + b_i

  Returns:
    init_fn, apply_fn, kernel_fn
  """
  # TODO(jaschasd) after experimentation, evaluate whether to change default
  # parameterization from "ntk" to "standard"

  parameterization = parameterization.lower()

  ntk_init_fn, _ = ostax.Dense(out_dim, W_init, b_init)

  def standard_init_fn(rng, input_shape):
    output_shape, (W, b) = ntk_init_fn(rng, input_shape)
    return output_shape, (W * W_std / np.sqrt(input_shape[-1]), b * b_std)

  if parameterization == 'ntk':
    init_fn = ntk_init_fn
  elif parameterization == 'standard':
    init_fn = standard_init_fn
  else:
    raise ValueError('Parameterization not supported: %s' % parameterization)

  def apply_fn(params, inputs, **kwargs):
    W, b = params
    if parameterization == 'ntk':
      norm = W_std / np.sqrt(inputs.shape[-1])
      return norm * np.dot(inputs, W) + b_std * b
    elif parameterization == 'standard':
      return np.dot(inputs, W) + b

  def kernel_fn(kernels):
    """Compute the transformed kernels after a dense layer."""
    var1, nngp, var2, ntk = \
        kernels.var1, kernels.nngp, kernels.var2, kernels.ntk

    def fc(x):
      return _affine(x, W_std, b_std)

    if parameterization == 'ntk':
      var1, nngp, var2 = map(fc, (var1, nngp, var2))
      if ntk is not None:
        ntk = nngp + W_std**2 * ntk
    elif parameterization == 'standard':
      input_width = kernels.shape1[1]
      if ntk is not None:
        ntk = input_width * nngp + 1. + W_std**2 * ntk
      var1, nngp, var2 = map(fc, (var1, nngp, var2))

    return kernels._replace(
        var1=var1, nngp=nngp, var2=var2, ntk=ntk, is_gaussian=True)

  setattr(kernel_fn, _COVARIANCES_REQ,
          {'marginal': M.OVER_ALL, 'cross': M.OVER_ALL})
  return init_fn, apply_fn, kernel_fn
r"""Layer constructor function for a dense (fully-connected) layer.

  Based on `jax.experimental.stax.Dense`. Has a similar API, apart from:

  W_init and b_init only change the behavior of the finite width network, and
    are not used by kernel_fn. In most cases, W_std and b_std should be used
    instead

  Args:
    parameterization: Either 'ntk' or 'standard'.
      Under ntk parameterization (https://arxiv.org/abs/1806.07572, page 3),
        weights and biases are initialized as W_ij ~ N(0,1), b_i ~ N(0,1), and
        the finite width layer equation is z_i = W_std / sqrt([width]) sum_j
        W_ij x_j + b_std b_i Under standard parameterization, weights and biases
        are initialized as W_ij ~ N(0,W_std^2/[width]), b_i ~ N(0,b_std^2), and
        the finite width layer equation is z_i = \sum_j W_ij x_j + b_i

  Returns:
    init_fn, apply_fn, kernel_fn
  
r"""Layer constructor function for a dense (fully-connected) layer.

W_init and b_init only change the behavior of the finite width network, and
are not used by kernel_fn. In most cases, W_std and b_std should be used
instead
r " " " Layer constructor function for a dense ( fully - connected ) layer . W_init and b_init only change the behavior of the finite width network and are not used by kernel_fn . In most cases W_std and b_std should be used instead
def Dense(out_dim, W_std=1., b_std=0., W_init=_randn(1.0), b_init=_randn(1.0),
          parameterization='ntk'):
  parameterization = parameterization.lower()
  ntk_init_fn, _ = ostax.Dense(out_dim, W_init, b_init)
  def standard_init_fn(rng, input_shape):
    output_shape, (W, b) = ntk_init_fn(rng, input_shape)
    return output_shape, (W * W_std / np.sqrt(input_shape[-1]), b * b_std)
  if parameterization == 'ntk':
    init_fn = ntk_init_fn
  elif parameterization == 'standard':
    init_fn = standard_init_fn
  else:
    raise ValueError('Parameterization not supported: %s' % parameterization)
  def apply_fn(params, inputs, **kwargs):
    W, b = params
    if parameterization == 'ntk':
      norm = W_std / np.sqrt(inputs.shape[-1])
      return norm * np.dot(inputs, W) + b_std * b
    elif parameterization == 'standard':
      return np.dot(inputs, W) + b
  def kernel_fn(kernels):
    var1, nngp, var2, ntk = \
        kernels.var1, kernels.nngp, kernels.var2, kernels.ntk
    def fc(x):
      return _affine(x, W_std, b_std)
    if parameterization == 'ntk':
      var1, nngp, var2 = map(fc, (var1, nngp, var2))
      if ntk is not None:
        ntk = nngp + W_std**2 * ntk
    elif parameterization == 'standard':
      input_width = kernels.shape1[1]
      if ntk is not None:
        ntk = input_width * nngp + 1. + W_std**2 * ntk
      var1, nngp, var2 = map(fc, (var1, nngp, var2))
    return kernels._replace(
        var1=var1, nngp=nngp, var2=var2, ntk=ntk, is_gaussian=True)
  setattr(kernel_fn, _COVARIANCES_REQ,
          {'marginal': M.OVER_ALL, 'cross': M.OVER_ALL})
  return init_fn, apply_fn, kernel_fn
def Dense ( out_dim , W_std = 1. , b_std = 0. , W_init = _randn ( 1.0 ) , b_init = _randn ( 1.0 ) , parameterization = 'ntk' ) : parameterization = parameterization . lower ( ) ntk_init_fn , _ = ostax . Dense ( out_dim , W_init , b_init ) def standard_init_fn ( rng , input_shape ) : output_shape , ( W , b ) = ntk_init_fn ( rng , input_shape ) return output_shape , ( W * W_std / np . sqrt ( input_shape [ - 1 ] ) , b * b_std ) if parameterization == 'ntk' : init_fn = ntk_init_fn elif parameterization == 'standard' : init_fn = standard_init_fn else : raise ValueError ( 'Parameterization not supported: %s' % parameterization ) def apply_fn ( params , inputs , ** kwargs ) : W , b = params if parameterization == 'ntk' : norm = W_std / np . sqrt ( inputs . shape [ - 1 ] ) return norm * np . dot ( inputs , W ) + b_std * b elif parameterization == 'standard' : return np . dot ( inputs , W ) + b def kernel_fn ( kernels ) : """Compute the transformed kernels after a dense layer.""" var1 , nngp , var2 , ntk = kernels . var1 , kernels . nngp , kernels . var2 , kernels . ntk def fc ( x ) : return _affine ( x , W_std , b_std ) if parameterization == 'ntk' : var1 , nngp , var2 = map ( fc , ( var1 , nngp , var2 ) ) if ntk is not None : ntk = nngp + W_std ** 2 * ntk elif parameterization == 'standard' : input_width = kernels . shape1 [ 1 ] if ntk is not None : ntk = input_width * nngp + 1. + W_std ** 2 * ntk var1 , nngp , var2 = map ( fc , ( var1 , nngp , var2 ) ) return kernels . _replace ( var1 = var1 , nngp = nngp , var2 = var2 , ntk = ntk , is_gaussian = True ) setattr ( kernel_fn , _COVARIANCES_REQ , { 'marginal' : M . OVER_ALL , 'cross' : M . OVER_ALL } ) return init_fn , apply_fn , kernel_fn
r"""Layer constructor function for a dense (fully-connected) layer.
r " " " Layer constructor function for a dense ( fully - connected ) layer .
r"""Layer constructor function for a dense (fully-connected) layer.

  Based on `jax.experimental.stax.Dense`. Has a similar API, apart from:

  W_init and b_init only change the behavior of the finite width network, and
    are not used by kernel_fn. In most cases, W_std and b_std should be used
    instead

  Args:
    parameterization: Either 'ntk' or 'standard'.
      Under ntk parameterization (https://arxiv.org/abs/1806.07572, page 3),
        weights and biases are initialized as W_ij ~ N(0,1), b_i ~ N(0,1), and
        the finite width layer equation is z_i = W_std / sqrt([width]) sum_j
        W_ij x_j + b_std b_i Under standard parameterization, weights and biases
        are initialized as W_ij ~ N(0,W_std^2/[width]), b_i ~ N(0,b_std^2), and
        the finite width layer equation is z_i = \sum_j W_ij x_j + b_i

  Returns:
    init_fn, apply_fn, kernel_fn
  """ # TODO(jaschasd) after experimentation, evaluate whether to change default # parameterization from "ntk" to "standard" """Compute the transformed kernels after a dense layer."""
None
out_dim
W_std
b_std
W_init
b_init
parameterization
Either 'ntk' or 'standard'.
Under ntk parameterization ,
weights and biases are initialized as W_ij ~ N(0,1), b_i ~ N(0,1), and
the finite width layer equation is z_i = W_std / sqrt([width]) sum_j
W_ij x_j + b_std b_i Under standard parameterization, weights and biases
are initialized as W_ij ~ N(0,W_std^2/[width]), b_i ~ N(0,b_std^2), and
the finite width layer equation is z_i = \sum_j W_ij x_j + b_i
Either ' ntk ' or ' standard ' . Under ntk parameterization weights and biases are initialized as W_ij ~ N ( 0 1 ) b_i ~ N ( 0 1 ) and the finite width layer equation is z_i = W_std / sqrt ( [ width ] ) sum_j W_ij x_j + b_std b_i Under standard parameterization weights and biases are initialized as W_ij ~ N ( 0 W_std^2 / [ width ] ) b_i ~ N ( 0 b_std^2 ) and the finite width layer equation is z_i = \ sum_j W_ij x_j + b_i