coordle/utils/utils.py
def clean_text(text: str, stopwords: set=None, punctuation: str=None,
               return_list: bool=True) -> Union[str, list]:
    '''
    Cleans text by turning to lower case, removing punctuations and stopwords
    
    Parameters:
    --------------
    text: String to be cleaned 
    
    stopwords: Optional set of stopwords (as strings). If None is given, the 
               stopwords will be acquired from 
               nltk.corpus.stopwords.words('english')
               
    punctuation: Optional string punctuations. If None is given, the 
                 punctuations will be acquired from strings.punctuations

    return_list: Optional, will return list of tokens of True (True by default),
                 else it returns a string
    
    Returns:
    --------
    If return_list is True: list
    else: string
    '''
    if stopwords is None:
        stopwords = set(_stopwords.words('english'))
        
    if punctuation is None:
        punctuation = PUNCTUATION

    # Lower case 
    text = text.lower()
    # Replace newlines with spaces
    text = text.replace('\n',' ')
    # Remove punctuations
    text = re.sub(f'[{punctuation}]','',text)

    # Filter function, remove stopwords, numbers
    # and strings that have length 1 or less
    f = lambda t: (t not in stopwords) and \
                  (not t.isnumeric()) and \
                  (len(t) > 1)

    # Lemmatize first, then filter
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(t) for t in text.split()]
    tokens = [t for t in tokens if f(t)]
    
    if return_list:
        return tokens
    else:
        return str.join(' ', tokens)

    Cleans text by turning to lower case, removing punctuations and stopwords
    
    Parameters:
    --------------
    text: String to be cleaned 
    
    stopwords: Optional set of stopwords (as strings). If None is given, the 
               stopwords will be acquired from 
               nltk.corpus.stopwords.words('english')
               
    punctuation: Optional string punctuations. If None is given, the 
                 punctuations will be acquired from strings.punctuations

    return_list: Optional, will return list of tokens of True (True by default),
                 else it returns a string
    
    Returns:
    --------
    If return_list is True: list
    else: string
    
Cleans text by turning to lower case, removing punctuations and stopwords
Parameters.

String to be cleaned

Optional set of stopwords (as strings). If None is given, the
stopwords will be acquired from
nltk.corpus.stopwords.words('english')

Optional string punctuations. If None is given, the
punctuations will be acquired from strings.punctuations

Optional, will return list of tokens of True (True by default),
else it returns a string



If return_list is True: list
else: string
Cleans text by turning to lower case removing punctuations and stopwords Parameters . String to be cleaned Optional set of stopwords ( as strings ) . If None is given the stopwords will be acquired from nltk . corpus . stopwords . words ( ' english ' ) Optional string punctuations . If None is given the punctuations will be acquired from strings . punctuations Optional will return list of tokens of True ( True by default ) else it returns a string If return_list is True : list else : string
def clean_text(text: str, stopwords: set=None, punctuation: str=None,
               return_list: bool=True) -> Union[str, list]:
    if stopwords is None:
        stopwords = set(_stopwords.words('english'))
    if punctuation is None:
        punctuation = PUNCTUATION
    text = text.lower()
    text = text.replace('\n',' ')
    text = re.sub(f'[{punctuation}]','',text)
    f = lambda t: (t not in stopwords) and \
                  (not t.isnumeric()) and \
                  (len(t) > 1)
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(t) for t in text.split()]
    tokens = [t for t in tokens if f(t)]
    if return_list:
        return tokens
    else:
        return str.join(' ', tokens)
def clean_text ( text : str , stopwords : set = None , punctuation : str = None , return_list : bool = True ) -> Union [ str , list ] : if stopwords is None : stopwords = set ( _stopwords . words ( 'english' ) ) if punctuation is None : punctuation = PUNCTUATION text = text . lower ( ) text = text . replace ( '\n' , ' ' ) text = re . sub ( f'[{punctuation}]' , '' , text ) f = lambda t : ( t not in stopwords ) and ( not t . isnumeric ( ) ) and ( len ( t ) > 1 ) lemmatizer = WordNetLemmatizer ( ) tokens = [ lemmatizer . lemmatize ( t ) for t in text . split ( ) ] tokens = [ t for t in tokens if f ( t ) ] if return_list : return tokens else : return str . join ( ' ' , tokens )
Cleans text by turning to lower case, removing punctuations and stopwords
Parameters:
Cleans text by turning to lower case removing punctuations and stopwords Parameters :
'''
    Cleans text by turning to lower case, removing punctuations and stopwords
    
    Parameters:
    --------------
    text: String to be cleaned 
    
    stopwords: Optional set of stopwords (as strings). If None is given, the 
               stopwords will be acquired from 
               nltk.corpus.stopwords.words('english')
               
    punctuation: Optional string punctuations. If None is given, the 
                 punctuations will be acquired from strings.punctuations

    return_list: Optional, will return list of tokens of True (True by default),
                 else it returns a string
    
    Returns:
    --------
    If return_list is True: list
    else: string
    ''' # Lower case  # Replace newlines with spaces # Remove punctuations # Filter function, remove stopwords, numbers # and strings that have length 1 or less # Lemmatize first, then filter
text
str
stopwords
set
punctuation
str
return_list
bool