data.py
def create_vocab(data_files, vocab_fname):
    """
    Creates the character vocabulary file from a
    text dataset. Adds special tokens
    """
    chars = set()
    for data_fname in data_files:
        with io.open(data_fname, 'r', encoding='utf8') as fp:
            raw = fp.read().lower()
            chars.update(raw)

    vocab = list(chars - set(['\t', '\n'])) + SPECIALS
    tf.compat.v1.logging.info('Creating vocab file..')
    with io.open(vocab_fname, 'w', encoding='utf8') as fp:
        fp.write('\n'.join(vocab))

    Creates the character vocabulary file from a
    text dataset. Adds special tokens
    
Creates the character vocabulary file from a
text dataset. Adds special tokens
Creates the character vocabulary file from a text dataset . Adds special tokens
def create_vocab(data_files, vocab_fname):
    chars = set()
    for data_fname in data_files:
        with io.open(data_fname, 'r', encoding='utf8') as fp:
            raw = fp.read().lower()
            chars.update(raw)
    vocab = list(chars - set(['\t', '\n'])) + SPECIALS
    tf.compat.v1.logging.info('Creating vocab file..')
    with io.open(vocab_fname, 'w', encoding='utf8') as fp:
        fp.write('\n'.join(vocab))
def create_vocab ( data_files , vocab_fname ) : chars = set ( ) for data_fname in data_files : with io . open ( data_fname , 'r' , encoding = 'utf8' ) as fp : raw = fp . read ( ) . lower ( ) chars . update ( raw ) vocab = list ( chars - set ( [ '\t' , '\n' ] ) ) + SPECIALS tf . compat . v1 . logging . info ( 'Creating vocab file..' ) with io . open ( vocab_fname , 'w' , encoding = 'utf8' ) as fp : fp . write ( '\n' . join ( vocab ) )
Creates the character vocabulary file from a
text dataset.
Creates the character vocabulary file from a text dataset .
"""
    Creates the character vocabulary file from a
    text dataset. Adds special tokens
    """
data_files
vocab_fname