fulltext/process/psv.py
def _clean_sentence(lines: List[str]) -> List[str]:
    """Remove non-alphabet from the sentences. Convert to lower-case."""
    out: List[str] = []
    for line in lines:
        # continue if the line does not have any words
        if not re.match(r'\w', line):
            continue

        # replace all non-alphabet to space
        line = re.subn(r'\W', ' ', line)[0]
        line = _remove_ExtraSpaces(line)

        # remove all space in the beginning and end of the sentence
        line = re.sub(r'^\s+', '', line)
        line = re.sub(r'\s+$', '', line)

        # Remove "sentences" that has less than or equal to 3 characters
        if len(line) <= 3:
            continue

        out.append(line.lower())
    return out
Remove non-alphabet from the sentences. Convert to lower-case.
Remove non-alphabet from the sentences. Convert to lower-case.
Remove non - alphabet from the sentences . Convert to lower - case .
def _clean_sentence(lines: List[str]) -> List[str]:
    out: List[str] = []
    for line in lines:
        if not re.match(r'\w', line):
            continue
        line = re.subn(r'\W', ' ', line)[0]
        line = _remove_ExtraSpaces(line)
        line = re.sub(r'^\s+', '', line)
        line = re.sub(r'\s+$', '', line)
        if len(line) <= 3:
            continue
        out.append(line.lower())
    return out
def _clean_sentence ( lines : List [ str ] ) -> List [ str ] : out : List [ str ] = [ ] for line in lines : if not re . match ( r'\w' , line ) : continue line = re . subn ( r'\W' , ' ' , line ) [ 0 ] line = _remove_ExtraSpaces ( line ) line = re . sub ( r'^\s+' , '' , line ) line = re . sub ( r'\s+$' , '' , line ) if len ( line ) <= 3 : continue out . append ( line . lower ( ) ) return out
Remove non-alphabet from the sentences.
Remove non - alphabet from the sentences .
"""Remove non-alphabet from the sentences. Convert to lower-case.""" # continue if the line does not have any words # replace all non-alphabet to space # remove all space in the beginning and end of the sentence # Remove "sentences" that has less than or equal to 3 characters
lines
List[str]