Jann/utils.py
def load_data(file_path, dest_type, pairs=False, delimiter='\t'):
    """Load line separated text files into list. """

    dest = None
    dest2 = None
    if dest_type == 'list':
        if not pairs:
            tempfile = io.open(
                file_path, 'r', encoding="iso-8859-1", errors='ignore')
            dest = []
            for line in tempfile:
                clean_string = line.strip()
                # check if blank
                if clean_string:
                    dest.append(clean_string)
            tempfile.close()
        else:
            first_lines = []
            second_lines = []
            tf.logging.info('Loading pairs data')
            with open(file_path, 'r', encoding='iso-8859-1') as f:
                reader = csv.reader(f, delimiter=delimiter)
                for row in reader:
                    first_lines.append(row[0])
                    second_lines.append(row[1])
            dest = first_lines
            dest2 = second_lines
    elif dest_type == 'dict':
        with open(file_path, 'rb') as f:
            dest = pickle.load(f)
    return dest, dest2
Load line separated text files into list. 
Load line separated text files into list.
Load line separated text files into list .
def load_data(file_path, dest_type, pairs=False, delimiter='\t'):
    dest = None
    dest2 = None
    if dest_type == 'list':
        if not pairs:
            tempfile = io.open(
                file_path, 'r', encoding="iso-8859-1", errors='ignore')
            dest = []
            for line in tempfile:
                clean_string = line.strip()
                if clean_string:
                    dest.append(clean_string)
            tempfile.close()
        else:
            first_lines = []
            second_lines = []
            tf.logging.info('Loading pairs data')
            with open(file_path, 'r', encoding='iso-8859-1') as f:
                reader = csv.reader(f, delimiter=delimiter)
                for row in reader:
                    first_lines.append(row[0])
                    second_lines.append(row[1])
            dest = first_lines
            dest2 = second_lines
    elif dest_type == 'dict':
        with open(file_path, 'rb') as f:
            dest = pickle.load(f)
    return dest, dest2
def load_data ( file_path , dest_type , pairs = False , delimiter = '\t' ) : dest = None dest2 = None if dest_type == 'list' : if not pairs : tempfile = io . open ( file_path , 'r' , encoding = "iso-8859-1" , errors = 'ignore' ) dest = [ ] for line in tempfile : clean_string = line . strip ( ) if clean_string : dest . append ( clean_string ) tempfile . close ( ) else : first_lines = [ ] second_lines = [ ] tf . logging . info ( 'Loading pairs data' ) with open ( file_path , 'r' , encoding = 'iso-8859-1' ) as f : reader = csv . reader ( f , delimiter = delimiter ) for row in reader : first_lines . append ( row [ 0 ] ) second_lines . append ( row [ 1 ] ) dest = first_lines dest2 = second_lines elif dest_type == 'dict' : with open ( file_path , 'rb' ) as f : dest = pickle . load ( f ) return dest , dest2
Load line separated text files into list.
Load line separated text files into list .
"""Load line separated text files into list. """ # check if blank
file_path
dest_type
pairs
delimiter