class_docstr: Multi-head attention.
  
  Example usage::
  
    >>> import flax.linen as nn
    >>> import jax
  
    >>> layer = nn.MultiHeadAttention(num_heads=8, qkv_features=16)
    >>> key1, key2, key3, key4, key5, key6 = jax.random.split(jax.random.key(0), 6)
    >>> shape = (4, 3, 2, 5)
    >>> q, k, v = jax.random.uniform(key1, shape), jax.random.uniform(key2, shape), jax.random.uniform(key3, shape)
    >>> variables = layer.init(jax.random.key(0), q)
  
    >>> # different inputs for inputs_q, inputs_k and inputs_v
    >>> out = layer.apply(variables, q, k, v)
    >>> # equivalent to layer.apply(variables, inputs_q=q, inputs_k=k, inputs_v=k)
    >>> out = layer.apply(variables, q, k)
    >>> # equivalent to layer.apply(variables, inputs_q=q, inputs_k=q) and layer.apply(variables, inputs_q=q, inputs_k=q, inputs_v=q)
    >>> out = layer.apply(variables, q)
  
    >>> attention_kwargs = dict(
    ...     num_heads=8,
    ...     qkv_features=16,
    ...     kernel_init=nn.initializers.ones,
    ...     bias_init=nn.initializers.zeros,
    ...     dropout_rate=0.5,
    ...     deterministic=False,
    ...     )
    >>> class Module(nn.Module):
    ...   attention_kwargs: dict
    ...
    ...   @nn.compact
    ...   def __call__(self, x, dropout_rng=None):
    ...     out1 = nn.MultiHeadAttention(**self.attention_kwargs)(x, dropout_rng=dropout_rng)
    ...     out2 = nn.MultiHeadAttention(**self.attention_kwargs)(x, dropout_rng=dropout_rng)
    ...     return out1, out2
    >>> module = Module(attention_kwargs)
    >>> variables = module.init({'params': key1, 'dropout': key2}, q)
  
    >>> # out1 and out2 are different.
    >>> out1, out2 = module.apply(variables, q, rngs={'dropout': key3})
    >>> # out3 and out4 are different.
    >>> # out1 and out3 are different. out2 and out4 are different.
    >>> out3, out4 = module.apply(variables, q, rngs={'dropout': key4})
    >>> # out1 and out2 are the same.
    >>> out1, out2 = module.apply(variables, q, dropout_rng=key5)
    >>> # out1 and out2 are the same as out3 and out4.
    >>> # providing a `dropout_rng` arg will take precedence over the `rngs` arg in `.apply`
    >>> out3, out4 = module.apply(variables, q, rngs={'dropout': key6}, dropout_rng=key5)
  
  Attributes:
    num_heads: number of attention heads. Features (i.e. inputs_q.shape[-1])
      should be divisible by the number of heads.
    dtype: the dtype of the computation (default: infer from inputs and params)
    param_dtype: the dtype passed to parameter initializers (default: float32)
    qkv_features: dimension of the key, query, and value.
    out_features: dimension of the last projection
    broadcast_dropout: bool: use a broadcasted dropout along batch dims.
    dropout_rate: dropout rate
    deterministic: if false, the attention weight is masked randomly using
      dropout, whereas if true, the attention weights are deterministic.
    precision: numerical precision of the computation see `jax.lax.Precision`
      for details.
    kernel_init: initializer for the kernel of the Dense layers.
    bias_init: initializer for the bias of the Dense layers.
    use_bias: bool: whether pointwise QKVO dense transforms use bias.
    attention_fn: dot_product_attention or compatible function. Accepts query,
      key, value, and returns output of shape `[bs, dim1, dim2, ..., dimN,,
      num_heads, value_channels]``
    decode: whether to prepare and use an autoregressive cache.
    normalize_qk: should QK normalization be applied (arxiv.org/abs/2302.05442).
class_docstr_tok: ["Multi-head","attention",".","Example","usage",":",":",">",">",">","import","flax.linen","as","nn",">",">",">","import","jax",">",">",">","layer","=","nn.MultiHeadAttention","(","num_heads=8",",","qkv_features=16",")",">",">",">","key1",",","key2",",","key3",",","key4",",","key5",",","key6","=","jax.random.split","(","jax.random.key","(","0",")",",","6",")",">",">",">","shape","=","(","4",",","3",",","2",",","5",")",">",">",">","q",",","k",",","v","=","jax.random.uniform","(","key1",",","shape",")",",","jax.random.uniform","(","key2",",","shape",")",",","jax.random.uniform","(","key3",",","shape",")",">",">",">","variables","=","layer.init","(","jax.random.key","(","0",")",",","q",")",">",">",">","#","different","inputs","for","inputs_q",",","inputs_k","and","inputs_v",">",">",">","out","=","layer.apply","(","variables",",","q",",","k",",","v",")",">",">",">","#","equivalent","to","layer.apply","(","variables",",","inputs_q=q",",","inputs_k=k",",","inputs_v=k",")",">",">",">","out","=","layer.apply","(","variables",",","q",",","k",")",">",">",">","#","equivalent","to","layer.apply","(","variables",",","inputs_q=q",",","inputs_k=q",")","and","layer.apply","(","variables",",","inputs_q=q",",","inputs_k=q",",","inputs_v=q",")",">",">",">","out","=","layer.apply","(","variables",",","q",")",">",">",">","attention_kwargs","=","dict","(","...","num_heads=8",",","...","qkv_features=16",",","...","kernel_init=nn.initializers.ones",",","...","bias_init=nn.initializers.zeros",",","...","dropout_rate=0.5",",","...","deterministic=False",",","...",")",">",">",">","class","Module","(","nn.Module",")",":","...","attention_kwargs",":","dict","...","...","@","nn.compact","...","def","__call__","(","self",",","x",",","dropout_rng=None",")",":","...","out1","=","nn.MultiHeadAttention","(","*","*","self.attention_kwargs",")","(","x",",","dropout_rng=dropout_rng",")","...","out2","=","nn.MultiHeadAttention","(","*","*","self.attention_kwargs",")","(","x",",","dropout_rng=dropout_rng",")","...","return","out1",",","out2",">",">",">","module","=","Module","(","attention_kwargs",")",">",">",">","variables","=","module.init","(","{","'params","'",":","key1",",","'dropout","'",":","key2","}",",","q",")",">",">",">","#","out1","and","out2","are","different",".",">",">",">","out1",",","out2","=","module.apply","(","variables",",","q",",","rngs=","{","'dropout","'",":","key3","}",")",">",">",">","#","out3","and","out4","are","different",".",">",">",">","#","out1","and","out3","are","different",".","out2","and","out4","are","different",".",">",">",">","out3",",","out4","=","module.apply","(","variables",",","q",",","rngs=","{","'dropout","'",":","key4","}",")",">",">",">","#","out1","and","out2","are","the","same",".",">",">",">","out1",",","out2","=","module.apply","(","variables",",","q",",","dropout_rng=key5",")",">",">",">","#","out1","and","out2","are","the","same","as","out3","and","out4",".",">",">",">","#","providing","a","`","dropout_rng","`","arg","will","take","precedence","over","the","`","rngs","`","arg","in","`",".apply","`",">",">",">","out3",",","out4","=","module.apply","(","variables",",","q",",","rngs=","{","'dropout","'",":","key6","}",",","dropout_rng=key5",")","Attributes",":","num_heads",":","number","of","attention","heads",".","Features","(","i.e",".","inputs_q.shape","[","-1","]",")","should","be","divisible","by","the","number","of","heads",".","dtype",":","the","dtype","of","the","computation","(","default",":","infer","from","inputs","and","params",")","param_dtype",":","the","dtype","passed","to","parameter","initializers","(","default",":","float32",")","qkv_features",":","dimension","of","the","key",",","query",",","and","value",".","out_features",":","dimension","of","the","last","projection","broadcast_dropout",":","bool",":","use","a","broadcasted","dropout","along","batch","dims",".","dropout_rate",":","dropout","rate","deterministic",":","if","false",",","the","attention","weight","is","masked","randomly","using","dropout",",","whereas","if","true",",","the","attention","weights","are","deterministic",".","precision",":","numerical","precision","of","the","computation","see","`","jax.lax.Precision","`","for","details",".","kernel_init",":","initializer","for","the","kernel","of","the","Dense","layers",".","bias_init",":","initializer","for","the","bias","of","the","Dense","layers",".","use_bias",":","bool",":","whether","pointwise","QKVO","dense","transforms","use","bias",".","attention_fn",":","dot_product_attention","or","compatible","function",".","Accepts","query",",","key",",","value",",","and","returns","output","of","shape","`","[","bs",",","dim1",",","dim2",",","...",",","dimN",",",",","num_heads",",","value_channels","]","``","decode",":","whether","to","prepare","and","use","an","autoregressive","cache",".","normalize_qk",":","should","QK","normalization","be","applied","(","arxiv.org\/abs\/2302.05442",")","."]
docstr: Applies multi-head dot product attention on the input data.
  
  Projects the inputs into multi-headed query, key, and value vectors,
  applies dot-product attention and project the results to an output vector.
  
  If both inputs_k and inputs_v are None, they will both copy the value of
  inputs_q (self attention).
  If only inputs_v is None, it will copy the value of inputs_k.
  
  Args:
    inputs_q: input queries of shape `[batch_sizes..., length, features]`.
    inputs_k: key of shape `[batch_sizes..., length, features]`. If None,
      inputs_k will copy the value of inputs_q.
    inputs_v: values of shape `[batch_sizes..., length, features]`. If None,
      inputs_v will copy the value of inputs_k.
    mask: attention mask of shape `[batch_sizes..., num_heads, query_length,
      key/value_length]`. Attention weights are masked out if their
      corresponding mask value is `False`.
    deterministic: if false, the attention weight is masked randomly using
      dropout, whereas if true, the attention weights are deterministic.
    rngs: container for random number generators to generate the dropout
      mask when `deterministic` is False. The `rngs` container should have a
      `dropout` key.
    sow_weights: if ``True``, the attention weights are sowed into the
      'intermediates' collection.
  
  Returns:
    output of shape `[batch_sizes..., length, features]`.
docstr_tok: ["Applies","multi-head","dot","product","attention","on","the","input","data",".","Projects","the","inputs","into","multi-headed","query",",","key",",","and","value","vectors",",","applies","dot-product","attention","and","project","the","results","to","an","output","vector",".","If","both","inputs_k","and","inputs_v","are","None",",","they","will","both","copy","the","value","of","inputs_q","(","self","attention",")",".","If","only","inputs_v","is","None",",","it","will","copy","the","value","of","inputs_k",".","Args",":","inputs_q",":","input","queries","of","shape","`","[","batch_sizes","...",",","length",",","features","]","`",".","inputs_k",":","key","of","shape","`","[","batch_sizes","...",",","length",",","features","]","`",".","If","None",",","inputs_k","will","copy","the","value","of","inputs_q",".","inputs_v",":","values","of","shape","`","[","batch_sizes","...",",","length",",","features","]","`",".","If","None",",","inputs_v","will","copy","the","value","of","inputs_k",".","mask",":","attention","mask","of","shape","`","[","batch_sizes","...",",","num_heads",",","query_length",",","key\/value_length","]","`",".","Attention","weights","are","masked","out","if","their","corresponding","mask","value","is","`","False","`",".","deterministic",":","if","false",",","the","attention","weight","is","masked","randomly","using","dropout",",","whereas","if","true",",","the","attention","weights","are","deterministic",".","rngs",":","container","for","random","number","generators","to","generate","the","dropout","mask","when","`","deterministic","`","is","False",".","The","`","rngs","`","container","should","have","a","`","dropout","`","key",".","sow_weights",":","if","``","True","``",",","the","attention","weights","are","sowed","into","the","'intermediates","'","collection",".","Returns",":","output","of","shape","`","[","batch_sizes","...",",","length",",","features","]","`","."]
code: def __call__(
      self,
      inputs_q: Array,
      inputs_k: Array | None = None,
      inputs_v: Array | None = None,
      *,
      mask: Array | None = None,
      deterministic: bool | None = None,
      rngs: rnglib.Rngs | None = None,
      sow_weights: bool = False,
      decode: bool | None = None,
  ):
      """Applies multi-head dot product attention on the input data.
  
      Projects the inputs into multi-headed query, key, and value vectors,
      applies dot-product attention and project the results to an output vector.
  
      If both inputs_k and inputs_v are None, they will both copy the value of
      inputs_q (self attention).
      If only inputs_v is None, it will copy the value of inputs_k.
  
      Args:
        inputs_q: input queries of shape `[batch_sizes..., length, features]`.
        inputs_k: key of shape `[batch_sizes..., length, features]`. If None,
          inputs_k will copy the value of inputs_q.
        inputs_v: values of shape `[batch_sizes..., length, features]`. If None,
          inputs_v will copy the value of inputs_k.
        mask: attention mask of shape `[batch_sizes..., num_heads, query_length,
          key/value_length]`. Attention weights are masked out if their
          corresponding mask value is `False`.
        deterministic: if false, the attention weight is masked randomly using
          dropout, whereas if true, the attention weights are deterministic.
        rngs: container for random number generators to generate the dropout
          mask when `deterministic` is False. The `rngs` container should have a
          `dropout` key.
        sow_weights: if ``True``, the attention weights are sowed into the
          'intermediates' collection.
  
      Returns:
        output of shape `[batch_sizes..., length, features]`.
      """
  
      if inputs_k is None:
          if inputs_v is not None:
              raise ValueError(
                  "`inputs_k` cannot be None if `inputs_v` is not None. "
                  "To have both `inputs_k` and `inputs_v` be the same value, pass in the "
                  "value to `inputs_k` and leave `inputs_v` as None."
              )
          inputs_k = inputs_q
      if inputs_v is None:
          inputs_v = inputs_k
  
      if inputs_q.shape[-1] != self.in_features:
          raise ValueError(
              f"Incompatible input dimension, got {inputs_q.shape[-1]} "
              f"but module expects {self.in_features}."
          )
  
      query = self.query(inputs_q)
      key = self.key(inputs_k)
      value = self.value(inputs_v)
  
      if self.normalize_qk:
          assert self.query_ln is not None and self.key_ln is not None
          # Normalizing query and key projections stabilizes training with higher
          # LR. See ViT-22B paper http://arxiv.org/abs/2302.05442 for analysis.
          query = self.query_ln(query)
          key = self.key_ln(key)
  
      # During fast autoregressive decoding, we feed one position at a time,
      # and cache the keys and values step by step.
      decode = first_from(
          decode,
          self.decode,
          flaglib.flags.get("decode"),
          error_msg="""No `decode` argument was provided to MultiHeadAttention
          as either a __call__ argument, class attribute, or nnx.flag.""",
      )
  
      if decode:
          (
              *batch_dims,
              max_length,
              num_heads,
              depth_per_head,
          ) = self.cached_key.shape
          # shape check of cached keys against query input
          expected_shape = tuple(batch_dims) + (
              1,
              num_heads,
              depth_per_head,
          )
          if expected_shape != query.shape:
              raise ValueError(
                  "Autoregressive cache shape error, "
                  "expected query shape %s instead got %s."
                  % (expected_shape, query.shape)
              )
          # update key, value caches with our new 1d spatial slices
          cur_index = self.cache_index
          zero = jnp.array(0, dtype=lax.dtype(cur_index.dtype))
          indices = (zero,) * len(batch_dims) + (cur_index, zero, zero)
          key = lax.dynamic_update_slice(self.cached_key, key, indices)
          value = lax.dynamic_update_slice(
              self.cached_value, value, indices
          )
          self.cached_key = key
          self.cached_value = value
          self.cache_index += 1
          # causal mask for cached decoder self-attention:
          # our single query position should only attend to those key
          # positions that have already been generated and cached,
          # not the remaining zero elements.
          mask = combine_masks(
              mask,
              jnp.broadcast_to(
                  jnp.arange(max_length) <= cur_index,
                  tuple(batch_dims) + (1, 1, max_length),
              ),
          )
  
      if (
          self.dropout_rate > 0.0
      ):  # Require `deterministic` only if using dropout.
          deterministic = first_from(
              deterministic,
              self.deterministic,
              flaglib.flags.get("deterministic"),
              error_msg="""No `deterministic` argument was provided to MultiHeadAttention
            as either a __call__ argument, class attribute, or nnx.flag.""",
          )
          if not deterministic:
              if rngs is None:
                  raise ValueError(
                      "'rngs' must be provided if 'dropout_rng' is not given."
                  )
              dropout_rng = rngs.dropout()
          else:
              dropout_rng = None
      else:
          deterministic = True
          dropout_rng = None
  
      # apply attention
      x = self.attention_fn(
          query,
          key,
          value,
          mask=mask,
          dropout_rng=dropout_rng,
          dropout_rate=self.dropout_rate,
          broadcast_dropout=self.broadcast_dropout,
          deterministic=deterministic,
          dtype=self.dtype,
          precision=self.precision,
          module=self if sow_weights else None,
      )
      # back to the original inputs dimensions
      out = self.out(x)
      return out
  
imports: from __future__ import annotations
  import functools
  from typing import Any, Callable, Optional, Tuple, overload
  import jax
  import jax.numpy
  from jax import lax, random
  from flax.experimental import nnx
  from flax.experimental.nnx.nnx import rnglib
  from flax.experimental.nnx.nnx import flaglib
  from flax.experimental.nnx.nnx.module import Module, first_from
  from flax.experimental.nnx.nnx.nn import initializers
  from flax.experimental.nnx.nnx.nn.dtypes import promote_dtype
  from flax.experimental.nnx.nnx.nn.linear import DotGeneralT, LinearGeneral, PrecisionLike, default_kernel_init
  from flax.experimental.nnx.nnx.nn.normalization import LayerNorm
label_desc: Use image node_id 4 for calling the MultiHeadAttention obj's underlying member method code with example usage: obj.__call__(inputs_q, inputs_k, inputs_v) and returns: out