lm1b/utils/vocab.py
def encode_sequence( seq, char_to_id_lookup_table, hparams ):
    """
    Encode strings to padded character vectors.

    Note: Original model didn't pad sequences, it was trained with one long sequences of sentences concated together.
    Will need to do something slightly different for re-training.

    :param seq: tensor of strings representing a sequence of words (e.g. a sentence). Should typically be wrapped with
    <S> and </S>, but this doesn't expect it.
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return:
    """

    sequence_length = hparams.sequence_length
    # Split by word
    seq = tf.string_split( seq )
    seq = tf.sparse_tensor_to_dense( seq, default_value=hparams.tokens_padding )
    # Pad to sequence length
    seq_padding = tf.fill( [sequence_length], tf.constant( hparams.tokens_padding ) )
    seq = tf.map_fn( lambda x: tf.concat( [x, seq_padding], 0 )[:sequence_length], seq )
    # Encode words to character vectors
    # set dtype since this transforms strings to int64s
    seq = tf.map_fn( lambda x: encode_token( x, char_to_id_lookup_table=char_to_id_lookup_table, hparams=hparams ), seq,
                     dtype=tf.int64 )
    return seq

    Encode strings to padded character vectors.

    Note: Original model didn't pad sequences, it was trained with one long sequences of sentences concated together.
    Will need to do something slightly different for re-training.

    :param seq: tensor of strings representing a sequence of words (e.g. a sentence). Should typically be wrapped with
    <S> and </S>, but this doesn't expect it.
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return:
    
Encode strings to padded character vectors.
Note: Original model didn't pad sequences, it was trained with one long sequences of sentences concated together.
Will need to do something slightly different for re-training.
Encode strings to padded character vectors . Note : Original model didn ' t pad sequences it was trained with one long sequences of sentences concated together . Will need to do something slightly different for re - training .
def encode_sequence( seq, char_to_id_lookup_table, hparams ):
    sequence_length = hparams.sequence_length
    seq = tf.string_split( seq )
    seq = tf.sparse_tensor_to_dense( seq, default_value=hparams.tokens_padding )
    seq_padding = tf.fill( [sequence_length], tf.constant( hparams.tokens_padding ) )
    seq = tf.map_fn( lambda x: tf.concat( [x, seq_padding], 0 )[:sequence_length], seq )
    seq = tf.map_fn( lambda x: encode_token( x, char_to_id_lookup_table=char_to_id_lookup_table, hparams=hparams ), seq,
                     dtype=tf.int64 )
    return seq
def encode_sequence ( seq , char_to_id_lookup_table , hparams ) : sequence_length = hparams . sequence_length seq = tf . string_split ( seq ) seq = tf . sparse_tensor_to_dense ( seq , default_value = hparams . tokens_padding ) seq_padding = tf . fill ( [ sequence_length ] , tf . constant ( hparams . tokens_padding ) ) seq = tf . map_fn ( lambda x : tf . concat ( [ x , seq_padding ] , 0 ) [ : sequence_length ] , seq ) seq = tf . map_fn ( lambda x : encode_token ( x , char_to_id_lookup_table = char_to_id_lookup_table , hparams = hparams ) , seq , dtype = tf . int64 ) return seq
Encode strings to padded character vectors.
Encode strings to padded character vectors .
"""
    Encode strings to padded character vectors.

    Note: Original model didn't pad sequences, it was trained with one long sequences of sentences concated together.
    Will need to do something slightly different for re-training.

    :param seq: tensor of strings representing a sequence of words (e.g. a sentence). Should typically be wrapped with
    <S> and </S>, but this doesn't expect it.
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return:
    """ # Split by word # Pad to sequence length # Encode words to character vectors # set dtype since this transforms strings to int64s
None
seq
tensor of strings representing a sequence of words . Should typically be wrapped with
 and , but this doesn't expect it.
tensor of strings representing a sequence of words . Should typically be wrapped with and but this doesn ' t expect it .
char_to_id_lookup_table
Lookup table mapping characters to ids. See utils.vocab for reference
Lookup table mapping characters to ids . See utils . vocab for reference
hparams
None