trax/models/rnn.py
def RNNLM(vocab_size,
          d_model=512,
          n_layers=2,
          rnn_cell=tl.LSTMCell,
          rnn_cell_d_state_multiplier=2,
          dropout=0.1,
          mode='train'):
  """Returns an RNN language model.

  The input to the model is a tensor of tokens (ints).

  Args:
    vocab_size: int: vocab size
    d_model: int:  depth of embedding (n_units in the RNN cell)
    n_layers: int: number of RNN layers
    rnn_cell: the RNN cell
    rnn_cell_d_state_multiplier: how many times is RNN cell state larger
    dropout: float: dropout rate (how much to drop out)
    mode: str: 'train', 'eval' or 'predict', predict mode is for fast inference

  Returns:
    An RNN language model as a layer that maps from a tensor of tokens
    to activations over a vocab set.
  """
  def MultiRNNCell():
    """Multi-layer RNN cell."""
    assert n_layers == 2
    return tl.Serial(
        tl.Parallel([], tl.Split(n_items=n_layers)),
        tl.SerialWithSideOutputs(
            [rnn_cell(n_units=d_model) for _ in range(n_layers)]),
        tl.Parallel([], tl.Concatenate(n_items=n_layers))
    )

  zero_state = tl.MakeZeroState(  # pylint: disable=no-value-for-parameter
      depth_multiplier=n_layers * rnn_cell_d_state_multiplier
  )

  return tl.Serial(
      tl.ShiftRight(mode=mode),
      tl.Embedding(vocab_size, d_model),
      tl.Dropout(rate=dropout, mode=mode),
      tl.Branch([], zero_state),
      tl.Scan(MultiRNNCell(), axis=1),
      tl.Select([0], n_in=2),  # Drop RNN state.
      tl.Dense(vocab_size),
      tl.LogSoftmax()
  )
Returns an RNN language model.

  The input to the model is a tensor of tokens (ints).

  Args:
    vocab_size: int: vocab size
    d_model: int:  depth of embedding (n_units in the RNN cell)
    n_layers: int: number of RNN layers
    rnn_cell: the RNN cell
    rnn_cell_d_state_multiplier: how many times is RNN cell state larger
    dropout: float: dropout rate (how much to drop out)
    mode: str: 'train', 'eval' or 'predict', predict mode is for fast inference

  Returns:
    An RNN language model as a layer that maps from a tensor of tokens
    to activations over a vocab set.
  
Returns an RNN language model.
The input to the model is a tensor of tokens (ints).
Returns an RNN language model . The input to the model is a tensor of tokens ( ints ) .
def RNNLM(vocab_size,
          d_model=512,
          n_layers=2,
          rnn_cell=tl.LSTMCell,
          rnn_cell_d_state_multiplier=2,
          dropout=0.1,
          mode='train'):
  def MultiRNNCell():
    assert n_layers == 2
    return tl.Serial(
        tl.Parallel([], tl.Split(n_items=n_layers)),
        tl.SerialWithSideOutputs(
            [rnn_cell(n_units=d_model) for _ in range(n_layers)]),
        tl.Parallel([], tl.Concatenate(n_items=n_layers))
    )
  zero_state = tl.MakeZeroState(  
      depth_multiplier=n_layers * rnn_cell_d_state_multiplier
  )
  return tl.Serial(
      tl.ShiftRight(mode=mode),
      tl.Embedding(vocab_size, d_model),
      tl.Dropout(rate=dropout, mode=mode),
      tl.Branch([], zero_state),
      tl.Scan(MultiRNNCell(), axis=1),
      tl.Select([0], n_in=2),  
      tl.Dense(vocab_size),
      tl.LogSoftmax()
  )
def RNNLM ( vocab_size , d_model = 512 , n_layers = 2 , rnn_cell = tl . LSTMCell , rnn_cell_d_state_multiplier = 2 , dropout = 0.1 , mode = 'train' ) : def MultiRNNCell ( ) : """Multi-layer RNN cell.""" assert n_layers == 2 return tl . Serial ( tl . Parallel ( [ ] , tl . Split ( n_items = n_layers ) ) , tl . SerialWithSideOutputs ( [ rnn_cell ( n_units = d_model ) for _ in range ( n_layers ) ] ) , tl . Parallel ( [ ] , tl . Concatenate ( n_items = n_layers ) ) ) zero_state = tl . MakeZeroState ( depth_multiplier = n_layers * rnn_cell_d_state_multiplier ) return tl . Serial ( tl . ShiftRight ( mode = mode ) , tl . Embedding ( vocab_size , d_model ) , tl . Dropout ( rate = dropout , mode = mode ) , tl . Branch ( [ ] , zero_state ) , tl . Scan ( MultiRNNCell ( ) , axis = 1 ) , tl . Select ( [ 0 ] , n_in = 2 ) , tl . Dense ( vocab_size ) , tl . LogSoftmax ( ) )
Returns an RNN language model.
Returns an RNN language model .
"""Returns an RNN language model.

  The input to the model is a tensor of tokens (ints).

  Args:
    vocab_size: int: vocab size
    d_model: int:  depth of embedding (n_units in the RNN cell)
    n_layers: int: number of RNN layers
    rnn_cell: the RNN cell
    rnn_cell_d_state_multiplier: how many times is RNN cell state larger
    dropout: float: dropout rate (how much to drop out)
    mode: str: 'train', 'eval' or 'predict', predict mode is for fast inference

  Returns:
    An RNN language model as a layer that maps from a tensor of tokens
    to activations over a vocab set.
  """ """Multi-layer RNN cell.""" # pylint: disable=no-value-for-parameter # Drop RNN state.
An RNN language model as a layer that maps from a tensor of tokens
to activations over a vocab set.
An RNN language model as a layer that maps from a tensor of tokens to activations over a vocab set .
vocab_size
None
d_model
depth of embedding (n_units in the RNN cell)
depth of embedding ( n_units in the RNN cell )
n_layers
number of RNN layers
number of RNN layers
rnn_cell
the RNN cell
the RNN cell
rnn_cell_d_state_multiplier
None
dropout
dropout rate (how much to drop out)
dropout rate ( how much to drop out )
mode
'train', 'eval' or 'predict', predict mode is for fast inference
' train ' ' eval ' or ' predict ' predict mode is for fast inference