lm1b/utils/vocab.py
def encode_token( token, char_to_id_lookup_table, hparams ):
    """
    Encode a word to a padded vector of character IDs

    :param token: tensor of strings representing single word / tokens
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return: tensor of shape [len(token), max_word_length], representing each word as a vector of character IDs
    """
    max_word_length = hparams.max_word_length
    chars_padding_id = hparams.chars_padding_id
    tokens_bos = hparams.tokens_bos  # begining of sentence
    tokens_eos = hparams.tokens_eos  # end of sentence
    chars_bow_id = hparams.chars_bow_id  # begining of word
    chars_eow_id = hparams.chars_eow_id  # end of word

    s = token
    # Special handling of sentence start and end tokens, per existing model
    s = tf.where( tf.equal( s, tokens_bos ), tf.fill( tf.shape( s ), chr( 0 ) ), s )
    s = tf.where( tf.equal( s, tokens_eos ), tf.fill( tf.shape( s ), chr( 1 ) ), s )
    # Add start and end of word symbols
    s = tf.map_fn( lambda x: tf.constant( chr( chars_bow_id ) ) + x + tf.constant( chr( chars_eow_id ) ), s )
    # Add padding
    padding_token = chr( chars_padding_id ) * max_word_length
    s = tf.map_fn( lambda x: tf.substr( x + tf.constant( padding_token ), 0, max_word_length ), s )
    # Split tokens into characters
    s = tf.sparse_tensor_to_dense( tf.string_split( s, delimiter="" ), default_value=chr( chars_padding_id ) )
    # Convert characters to char IDs
    s = char_to_id_lookup_table.lookup( s )
    return s

    Encode a word to a padded vector of character IDs

    :param token: tensor of strings representing single word / tokens
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return: tensor of shape [len(token), max_word_length], representing each word as a vector of character IDs
    
Encode a word to a padded vector of character IDs
Encode a word to a padded vector of character IDs
def encode_token( token, char_to_id_lookup_table, hparams ):
    max_word_length = hparams.max_word_length
    chars_padding_id = hparams.chars_padding_id
    tokens_bos = hparams.tokens_bos  
    tokens_eos = hparams.tokens_eos  
    chars_bow_id = hparams.chars_bow_id  
    chars_eow_id = hparams.chars_eow_id  
    s = token
    s = tf.where( tf.equal( s, tokens_bos ), tf.fill( tf.shape( s ), chr( 0 ) ), s )
    s = tf.where( tf.equal( s, tokens_eos ), tf.fill( tf.shape( s ), chr( 1 ) ), s )
    s = tf.map_fn( lambda x: tf.constant( chr( chars_bow_id ) ) + x + tf.constant( chr( chars_eow_id ) ), s )
    padding_token = chr( chars_padding_id ) * max_word_length
    s = tf.map_fn( lambda x: tf.substr( x + tf.constant( padding_token ), 0, max_word_length ), s )
    s = tf.sparse_tensor_to_dense( tf.string_split( s, delimiter="" ), default_value=chr( chars_padding_id ) )
    s = char_to_id_lookup_table.lookup( s )
    return s
def encode_token ( token , char_to_id_lookup_table , hparams ) : max_word_length = hparams . max_word_length chars_padding_id = hparams . chars_padding_id tokens_bos = hparams . tokens_bos tokens_eos = hparams . tokens_eos chars_bow_id = hparams . chars_bow_id chars_eow_id = hparams . chars_eow_id s = token s = tf . where ( tf . equal ( s , tokens_bos ) , tf . fill ( tf . shape ( s ) , chr ( 0 ) ) , s ) s = tf . where ( tf . equal ( s , tokens_eos ) , tf . fill ( tf . shape ( s ) , chr ( 1 ) ) , s ) s = tf . map_fn ( lambda x : tf . constant ( chr ( chars_bow_id ) ) + x + tf . constant ( chr ( chars_eow_id ) ) , s ) padding_token = chr ( chars_padding_id ) * max_word_length s = tf . map_fn ( lambda x : tf . substr ( x + tf . constant ( padding_token ) , 0 , max_word_length ) , s ) s = tf . sparse_tensor_to_dense ( tf . string_split ( s , delimiter = "" ) , default_value = chr ( chars_padding_id ) ) s = char_to_id_lookup_table . lookup ( s ) return s
Encode a word to a padded vector of character IDs
Encode a word to a padded vector of character IDs
"""
    Encode a word to a padded vector of character IDs

    :param token: tensor of strings representing single word / tokens
    :param char_to_id_lookup_table: Lookup table mapping characters to ids. See utils.vocab for reference
    :param hparams:
    :return: tensor of shape [len(token), max_word_length], representing each word as a vector of character IDs
    """ # begining of sentence # end of sentence # begining of word # end of word # Special handling of sentence start and end tokens, per existing model # Add start and end of word symbols # Add padding # Split tokens into characters # Convert characters to char IDs
tensor of shape [len(token), max_word_length], representing each word as a vector of character IDs
tensor of shape [ len ( token ) max_word_length ] representing each word as a vector of character IDs
token
tensor of strings representing single word / tokens
tensor of strings representing single word / tokens
char_to_id_lookup_table
Lookup table mapping characters to ids. See utils.vocab for reference
Lookup table mapping characters to ids . See utils . vocab for reference
hparams
None