Jann/process_novel_first_lines.py
def filter_line(line):
    """Filter lines so that we have a unique set of English-language lines."""
    # Simple clean-up rules for trailing characters and punctuation.
    line = line.strip()
    line = line.rstrip()
    line = line.replace('\t', ' ')
    line = line.replace('  ', ' ')
    line = line.replace('  ', ' ')
    line = line.replace('”', '"')
    line = line.replace('„', '"')
    line = line.replace('…', '...')
    # Remove lines where more than half of characters are not ASCII.
    num_chars = len(line)
    num_chars_non_ascii = 0
    for c in line:
        try:
            c.encode(encoding='utf-8').decode('ascii')
        except UnicodeDecodeError:
            num_chars_non_ascii += 1
    if num_chars_non_ascii * 2 > num_chars:
        return None
    # Store only unique lines.
    if line not in unique_lines:
        unique_lines.add(line)
        return line
    else:
        return None
Filter lines so that we have a unique set of English-language lines.
Filter lines so that we have a unique set of English-language lines.
Filter lines so that we have a unique set of English - language lines .
def filter_line(line):
    line = line.strip()
    line = line.rstrip()
    line = line.replace('\t', ' ')
    line = line.replace('  ', ' ')
    line = line.replace('  ', ' ')
    line = line.replace('”', '"')
    line = line.replace('„', '"')
    line = line.replace('…', '...')
    num_chars = len(line)
    num_chars_non_ascii = 0
    for c in line:
        try:
            c.encode(encoding='utf-8').decode('ascii')
        except UnicodeDecodeError:
            num_chars_non_ascii += 1
    if num_chars_non_ascii * 2 > num_chars:
        return None
    if line not in unique_lines:
        unique_lines.add(line)
        return line
    else:
        return None
def filter_line ( line ) : line = line . strip ( ) line = line . rstrip ( ) line = line . replace ( '\t' , ' ' ) line = line . replace ( '  ' , ' ' ) line = line . replace ( '  ' , ' ' ) line = line . replace ( '”',  ' ') line = line . replace ( '„',  ' ') line = line . replace ( '…',  ' ..') num_chars = len ( line ) num_chars_non_ascii = 0 for c in line : try : c . encode ( encoding = 'utf-8' ) . decode ( 'ascii' ) except UnicodeDecodeError : num_chars_non_ascii += 1 if num_chars_non_ascii * 2 > num_chars : return None if line not in unique_lines : unique_lines . add ( line ) return line else : return None
Filter lines so that we have a unique set of English-language lines.
Filter lines so that we have a unique set of English - language lines .
"""Filter lines so that we have a unique set of English-language lines.""" # Simple clean-up rules for trailing characters and punctuation. # Remove lines where more than half of characters are not ASCII. # Store only unique lines.
line