data.py
def split_text_file(data_file, model_dir, eval_fraction):
    """
    Split a Text dataset into train and evaluation
    """
    with io.open(data_file, 'r', encoding='utf-8') as fp:
        data = fp.readlines()

    random.shuffle(data)

    root, ext = os.path.splitext(data_file)
    train_file = os.path.join(model_dir, "{}-train{}".format(root, ext))
    eval_file = os.path.join(model_dir,"{}-eval{}".format(root, ext))
    train_offset = int(len(data)*(1-eval_fraction))

    if not os.path.exists(train_file) or not os.path.exists(eval_file):
        tf.compat.v1.logging.info('Splitting into train and test datasets..')
        with io.open(train_file, 'w', encoding='utf-8') as tfp,\
            io.open(eval_file, 'w', encoding='utf-8') as efp:

            for i, line in enumerate(data):
                if i < train_offset:
                    tfp.write(line)
                else:
                    efp.write(line)

    return train_file, eval_file

    Split a Text dataset into train and evaluation
    
Split a Text dataset into train and evaluation
Split a Text dataset into train and evaluation
def split_text_file(data_file, model_dir, eval_fraction):
    with io.open(data_file, 'r', encoding='utf-8') as fp:
        data = fp.readlines()
    random.shuffle(data)
    root, ext = os.path.splitext(data_file)
    train_file = os.path.join(model_dir, "{}-train{}".format(root, ext))
    eval_file = os.path.join(model_dir,"{}-eval{}".format(root, ext))
    train_offset = int(len(data)*(1-eval_fraction))
    if not os.path.exists(train_file) or not os.path.exists(eval_file):
        tf.compat.v1.logging.info('Splitting into train and test datasets..')
        with io.open(train_file, 'w', encoding='utf-8') as tfp,\
            io.open(eval_file, 'w', encoding='utf-8') as efp:
            for i, line in enumerate(data):
                if i < train_offset:
                    tfp.write(line)
                else:
                    efp.write(line)
    return train_file, eval_file
def split_text_file ( data_file , model_dir , eval_fraction ) : with io . open ( data_file , 'r' , encoding = 'utf-8' ) as fp : data = fp . readlines ( ) random . shuffle ( data ) root , ext = os . path . splitext ( data_file ) train_file = os . path . join ( model_dir , "{}-train{}" . format ( root , ext ) ) eval_file = os . path . join ( model_dir , "{}-eval{}" . format ( root , ext ) ) train_offset = int ( len ( data ) * ( 1 - eval_fraction ) ) if not os . path . exists ( train_file ) or not os . path . exists ( eval_file ) : tf . compat . v1 . logging . info ( 'Splitting into train and test datasets..' ) with io . open ( train_file , 'w' , encoding = 'utf-8' ) as tfp , io . open ( eval_file , 'w' , encoding = 'utf-8' ) as efp : for i , line in enumerate ( data ) : if i < train_offset : tfp . write ( line ) else : efp . write ( line ) return train_file , eval_file
Split a Text dataset into train and evaluation
Split a Text dataset into train and evaluation
"""
    Split a Text dataset into train and evaluation
    """
data_file
model_dir
eval_fraction