neural_tangents/stax.py
def _inputs_to_kernel(x1, x2, marginal, cross, compute_ntk):
  """Transforms (batches of) inputs to a `Kernel`.

  This is a private method. Docstring and example are for internal reference.

   The kernel contains the empirical covariances between different inputs and
     their entries (pixels) necessary to compute the covariance of the Gaussian
     Process corresponding to an infinite Bayesian or continuous gradient
     descent trained neural network.

   The smallest necessary number of covariance entries is tracked. For example,
     all networks are assumed to have i.i.d. weights along the channel / feature
     / logits dimensions, hence covariance between different entries along these
     dimensions is known to be 0 and is not tracked.

  Args:
    x1: a 2D `np.ndarray` of shape `[batch_size_1, n_features]` (dense
      network) or 4D of shape `[batch_size_1, height, width, channels]`
      (conv-nets).
    x2: an optional `np.ndarray` with the same shape as `x1` apart
      from possibly different leading batch size. `None` means
      `x2 == x1`.
    marginal: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `var1`/`var2`.
    cross: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `nngp`/`ntk`.
    compute_ntk: a boolean, `True` to compute both NTK and NNGP kernels,
        `False` to only compute NNGP.

    Example:
      ```python
          >>> x = np.ones((10, 32, 16, 3))
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_POINTS,
          >>>                       cross=M.NO,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 32, 16, 16), (10, 10, 32, 32, 16, 16)
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_PIXELS,
          >>>                       cross=M.OVER_PIXELS,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 16), (10, 10, 32, 16)
          >>> x1 = np.ones((10, 128))
          >>> x2 = np.ones((20, 128))
          >>> o = _inputs_to_kernel(x1, x2,
          >>>                       marginal=M.OVER_ALL,
          >>>                       cross=M.OVER_ALL,
          >>>                       compute_ntk=False)
          >>> o.var1.shape, o.nngp.shape
          (10,), (10, 20)
          >>> o.ntk
          None
      ```

  Returns:
    a `Kernel` object.
  """
  if x1.ndim not in (2, 4):
    raise ValueError('Inputs must be 2D or 4D `np.ndarray`s of shape '
                     '`[batch_size, n_features]` or '
                     '`[batch_size, height, width, channels]`, '
                     'got %s.' % str(x1.shape))

  if x1.ndim == 2 and not (marginal == cross == M.OVER_ALL):
    raise ValueError('`OVER_ALL` marginalisation should be used for 2D inputs; '
                     'was: `marginal`={}, `cross`={}'.format(marginal, cross))

  if cross == marginal == M.OVER_ALL:
    x1 = np.reshape(x1, (x1.shape[0], -1))
    if x2 is not None:
      x2 = np.reshape(x2, (x2.shape[0], -1))

  if cross == M.OVER_POINTS:
    raise ValueError('Required `OVER_POINTS` to be computed for `nngp`/`ntk`. '
                     '`OVER_POINTS` is only meant for `var1`/`var2`. '
                     'Use `NO` instead to compute all covariances.')

  # TODO(schsam, romann): Think more about dtype automatic vs manual dtype
  # promotion.
  x1 = x1.astype(np.float64)
  var1 = _get_variance(x1, marginal_type=marginal)

  if x2 is None:
    x2 = x1
    var2 = None
  else:
    if x1.shape[1:] != x2.shape[1:]:
      raise ValueError('`x1` and `x2` are expected to be batches of inputs'
                       ' with the same shape (apart from the batch size),'
                       ' got %s and %s.' %
                       (str(x1.shape), str(x2.shape)))

    x2 = x2.astype(np.float64)
    var2 = _get_variance(x2, marginal_type=marginal)

  nngp = _get_covariance(x1, x2, marginal_type=cross)
  ntk = 0. if compute_ntk else None
  is_gaussian = False
  is_height_width = True

  return Kernel(var1, nngp, var2, ntk, is_gaussian, is_height_width,
                marginal, cross, x1.shape, x2.shape)
Transforms (batches of) inputs to a `Kernel`.

  This is a private method. Docstring and example are for internal reference.

   The kernel contains the empirical covariances between different inputs and
     their entries (pixels) necessary to compute the covariance of the Gaussian
     Process corresponding to an infinite Bayesian or continuous gradient
     descent trained neural network.

   The smallest necessary number of covariance entries is tracked. For example,
     all networks are assumed to have i.i.d. weights along the channel / feature
     / logits dimensions, hence covariance between different entries along these
     dimensions is known to be 0 and is not tracked.

  Args:
    x1: a 2D `np.ndarray` of shape `[batch_size_1, n_features]` (dense
      network) or 4D of shape `[batch_size_1, height, width, channels]`
      (conv-nets).
    x2: an optional `np.ndarray` with the same shape as `x1` apart
      from possibly different leading batch size. `None` means
      `x2 == x1`.
    marginal: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `var1`/`var2`.
    cross: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `nngp`/`ntk`.
    compute_ntk: a boolean, `True` to compute both NTK and NNGP kernels,
        `False` to only compute NNGP.

    Example:
      ```python
          >>> x = np.ones((10, 32, 16, 3))
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_POINTS,
          >>>                       cross=M.NO,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 32, 16, 16), (10, 10, 32, 32, 16, 16)
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_PIXELS,
          >>>                       cross=M.OVER_PIXELS,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 16), (10, 10, 32, 16)
          >>> x1 = np.ones((10, 128))
          >>> x2 = np.ones((20, 128))
          >>> o = _inputs_to_kernel(x1, x2,
          >>>                       marginal=M.OVER_ALL,
          >>>                       cross=M.OVER_ALL,
          >>>                       compute_ntk=False)
          >>> o.var1.shape, o.nngp.shape
          (10,), (10, 20)
          >>> o.ntk
          None
      ```

  Returns:
    a `Kernel` object.
  
Transforms (batches of) inputs to a `Kernel`.
This is a private method. Docstring and example are for internal reference.

The kernel contains the empirical covariances between different inputs and
their entries (pixels) necessary to compute the covariance of the Gaussian
Process corresponding to an infinite Bayesian or continuous gradient
descent trained neural network.

The smallest necessary number of covariance entries is tracked. For example,
all networks are assumed to have i.i.d. weights along the channel / feature
logits dimensions, hence covariance between different entries along these
dimensions is known to be 0 and is not tracked.
Transforms ( batches of ) inputs to a ` Kernel ` . This is a private method . Docstring and example are for internal reference . The kernel contains the empirical covariances between different inputs and their entries ( pixels ) necessary to compute the covariance of the Gaussian Process corresponding to an infinite Bayesian or continuous gradient descent trained neural network . The smallest necessary number of covariance entries is tracked . For example all networks are assumed to have i . i . d . weights along the channel / feature logits dimensions hence covariance between different entries along these dimensions is known to be 0 and is not tracked .
def _inputs_to_kernel(x1, x2, marginal, cross, compute_ntk):
  if x1.ndim not in (2, 4):
    raise ValueError('Inputs must be 2D or 4D `np.ndarray`s of shape '
                     '`[batch_size, n_features]` or '
                     '`[batch_size, height, width, channels]`, '
                     'got %s.' % str(x1.shape))
  if x1.ndim == 2 and not (marginal == cross == M.OVER_ALL):
    raise ValueError('`OVER_ALL` marginalisation should be used for 2D inputs; '
                     'was: `marginal`={}, `cross`={}'.format(marginal, cross))
  if cross == marginal == M.OVER_ALL:
    x1 = np.reshape(x1, (x1.shape[0], -1))
    if x2 is not None:
      x2 = np.reshape(x2, (x2.shape[0], -1))
  if cross == M.OVER_POINTS:
    raise ValueError('Required `OVER_POINTS` to be computed for `nngp`/`ntk`. '
                     '`OVER_POINTS` is only meant for `var1`/`var2`. '
                     'Use `NO` instead to compute all covariances.')
  x1 = x1.astype(np.float64)
  var1 = _get_variance(x1, marginal_type=marginal)
  if x2 is None:
    x2 = x1
    var2 = None
  else:
    if x1.shape[1:] != x2.shape[1:]:
      raise ValueError('`x1` and `x2` are expected to be batches of inputs'
                       ' with the same shape (apart from the batch size),'
                       ' got %s and %s.' %
                       (str(x1.shape), str(x2.shape)))
    x2 = x2.astype(np.float64)
    var2 = _get_variance(x2, marginal_type=marginal)
  nngp = _get_covariance(x1, x2, marginal_type=cross)
  ntk = 0. if compute_ntk else None
  is_gaussian = False
  is_height_width = True
  return Kernel(var1, nngp, var2, ntk, is_gaussian, is_height_width,
                marginal, cross, x1.shape, x2.shape)
def _inputs_to_kernel ( x1 , x2 , marginal , cross , compute_ntk ) : if x1 . ndim not in ( 2 , 4 ) : raise ValueError ( 'Inputs must be 2D or 4D `np.ndarray`s of shape ' '`[batch_size, n_features]` or ' '`[batch_size, height, width, channels]`, ' 'got %s.' % str ( x1 . shape ) ) if x1 . ndim == 2 and not ( marginal == cross == M . OVER_ALL ) : raise ValueError ( '`OVER_ALL` marginalisation should be used for 2D inputs; ' 'was: `marginal`={}, `cross`={}' . format ( marginal , cross ) ) if cross == marginal == M . OVER_ALL : x1 = np . reshape ( x1 , ( x1 . shape [ 0 ] , - 1 ) ) if x2 is not None : x2 = np . reshape ( x2 , ( x2 . shape [ 0 ] , - 1 ) ) if cross == M . OVER_POINTS : raise ValueError ( 'Required `OVER_POINTS` to be computed for `nngp`/`ntk`. ' '`OVER_POINTS` is only meant for `var1`/`var2`. ' 'Use `NO` instead to compute all covariances.' ) x1 = x1 . astype ( np . float64 ) var1 = _get_variance ( x1 , marginal_type = marginal ) if x2 is None : x2 = x1 var2 = None else : if x1 . shape [ 1 : ] != x2 . shape [ 1 : ] : raise ValueError ( '`x1` and `x2` are expected to be batches of inputs' ' with the same shape (apart from the batch size),' ' got %s and %s.' % ( str ( x1 . shape ) , str ( x2 . shape ) ) ) x2 = x2 . astype ( np . float64 ) var2 = _get_variance ( x2 , marginal_type = marginal ) nngp = _get_covariance ( x1 , x2 , marginal_type = cross ) ntk = 0. if compute_ntk else None is_gaussian = False is_height_width = True return Kernel ( var1 , nngp , var2 , ntk , is_gaussian , is_height_width , marginal , cross , x1 . shape , x2 . shape )
Transforms (batches of) inputs to a `Kernel`.
Transforms ( batches of ) inputs to a ` Kernel ` .
"""Transforms (batches of) inputs to a `Kernel`.

  This is a private method. Docstring and example are for internal reference.

   The kernel contains the empirical covariances between different inputs and
     their entries (pixels) necessary to compute the covariance of the Gaussian
     Process corresponding to an infinite Bayesian or continuous gradient
     descent trained neural network.

   The smallest necessary number of covariance entries is tracked. For example,
     all networks are assumed to have i.i.d. weights along the channel / feature
     / logits dimensions, hence covariance between different entries along these
     dimensions is known to be 0 and is not tracked.

  Args:
    x1: a 2D `np.ndarray` of shape `[batch_size_1, n_features]` (dense
      network) or 4D of shape `[batch_size_1, height, width, channels]`
      (conv-nets).
    x2: an optional `np.ndarray` with the same shape as `x1` apart
      from possibly different leading batch size. `None` means
      `x2 == x1`.
    marginal: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `var1`/`var2`.
    cross: an instance of `Marginalisation` specifying for which spatial
      dimensions should the covariances be tracked in `nngp`/`ntk`.
    compute_ntk: a boolean, `True` to compute both NTK and NNGP kernels,
        `False` to only compute NNGP.

    Example:
      ```python
          >>> x = np.ones((10, 32, 16, 3))
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_POINTS,
          >>>                       cross=M.NO,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 32, 16, 16), (10, 10, 32, 32, 16, 16)
          >>> o = _inputs_to_kernel(x, None,
          >>>                       marginal=M.OVER_PIXELS,
          >>>                       cross=M.OVER_PIXELS,
          >>>                       compute_ntk=True)
          >>> o.var1.shape, o.ntk.shape
          (10, 32, 16), (10, 10, 32, 16)
          >>> x1 = np.ones((10, 128))
          >>> x2 = np.ones((20, 128))
          >>> o = _inputs_to_kernel(x1, x2,
          >>>                       marginal=M.OVER_ALL,
          >>>                       cross=M.OVER_ALL,
          >>>                       compute_ntk=False)
          >>> o.var1.shape, o.nngp.shape
          (10,), (10, 20)
          >>> o.ntk
          None
      ```

  Returns:
    a `Kernel` object.
  """ # TODO(schsam, romann): Think more about dtype automatic vs manual dtype # promotion.
a `Kernel` object.
a ` Kernel ` object .
x1
a 2D `np.ndarray` of shape `[batch_size_1, n_features]` (dense
network) or 4D of shape `[batch_size_1, height, width, channels]`
(conv-nets).
a 2D ` np . ndarray ` of shape ` [ batch_size_1 n_features ] ` ( dense network ) or 4D of shape ` [ batch_size_1 height width channels ] ` ( conv - nets ) .
x2
an optional `np.ndarray` with the same shape as `x1` apart
from possibly different leading batch size.
an optional ` np . ndarray ` with the same shape as ` x1 ` apart from possibly different leading batch size .
marginal
an instance of `Marginalisation` specifying for which spatial
dimensions should the covariances be tracked in `var1`/`var2`.
an instance of ` Marginalisation ` specifying for which spatial dimensions should the covariances be tracked in ` var1 ` / ` var2 ` .
cross
an instance of `Marginalisation` specifying for which spatial
dimensions should the covariances be tracked in `nngp`/`ntk`.
an instance of ` Marginalisation ` specifying for which spatial dimensions should the covariances be tracked in ` nngp ` / ` ntk ` .
compute_ntk
a boolean, `True` to compute both NTK and NNGP kernels,
`False` to only compute NNGP.
a boolean ` True ` to compute both NTK and NNGP kernels ` False ` to only compute NNGP .
Example
None