trax/models/rnn.py
def PrepareAttentionInputs():
    """Layer that prepares queries, keys, values and mask for attention."""
    def F(encoder_activations, decoder_activations, input_tokens):
      keys = values = encoder_activations
      queries = decoder_activations
      # Mask is 1 where inputs are not padding (0) and 0 where they are padding.
      mask = (input_tokens != 0)
      # We need to add axes to the mask for attention heads and decoder length.
      mask = jnp.reshape(mask, (mask.shape[0], 1, 1, mask.shape[1]))
      # Broadcast so mask is [batch, 1 for heads, decoder-len, encoder-len].
      mask = mask + jnp.zeros((1, 1, decoder_activations.shape[1], 1))
      return queries, keys, values, mask
    return tl.Fn('PrepareAttentionInputs', F, n_out=4)
Layer that prepares queries, keys, values and mask for attention.
Layer that prepares queries, keys, values and mask for attention.
Layer that prepares queries keys values and mask for attention .
def PrepareAttentionInputs():
    def F(encoder_activations, decoder_activations, input_tokens):
      keys = values = encoder_activations
      queries = decoder_activations
      mask = (input_tokens != 0)
      mask = jnp.reshape(mask, (mask.shape[0], 1, 1, mask.shape[1]))
      mask = mask + jnp.zeros((1, 1, decoder_activations.shape[1], 1))
      return queries, keys, values, mask
    return tl.Fn('PrepareAttentionInputs', F, n_out=4)
def PrepareAttentionInputs ( ) : def F ( encoder_activations , decoder_activations , input_tokens ) : keys = values = encoder_activations queries = decoder_activations mask = ( input_tokens != 0 ) mask = jnp . reshape ( mask , ( mask . shape [ 0 ] , 1 , 1 , mask . shape [ 1 ] ) ) mask = mask + jnp . zeros ( ( 1 , 1 , decoder_activations . shape [ 1 ] , 1 ) ) return queries , keys , values , mask return tl . Fn ( 'PrepareAttentionInputs' , F , n_out = 4 )
Layer that prepares queries, keys, values and mask for attention.
Layer that prepares queries keys values and mask for attention .
"""Layer that prepares queries, keys, values and mask for attention.""" # Mask is 1 where inputs are not padding (0) and 0 where they are padding. # We need to add axes to the mask for attention heads and decoder length. # Broadcast so mask is [batch, 1 for heads, decoder-len, encoder-len].