nlpmodels/utils/tokenizer.py
def tokenize_corpus_basic(text: List[str], removal: bool = True) -> List[List[str]]:
    """
    Function to tokenize a list of strings (corpus).

    Args:
        text (str): a list of strings to be converted into tokens.
        removal (bool): remove token list if it is empty
    Returns:
        a list of a list of tokens.
    """
    tokenizer = BasicEnglishTokenizer()
    new_text = []

    for line in text:
        # tokenize sentence
        tokens = [x for x in tokenizer.tokenize(line) if x not in string.punctuation]
        if not tokens and removal:
            continue
        new_text.append(tokens)

    return new_text

    Function to tokenize a list of strings (corpus).

    Args:
        text (str): a list of strings to be converted into tokens.
        removal (bool): remove token list if it is empty
    Returns:
        a list of a list of tokens.
    
Function to tokenize a list of strings (corpus).
Function to tokenize a list of strings ( corpus ) .
def tokenize_corpus_basic(text: List[str], removal: bool = True) -> List[List[str]]:
    tokenizer = BasicEnglishTokenizer()
    new_text = []
    for line in text:
        tokens = [x for x in tokenizer.tokenize(line) if x not in string.punctuation]
        if not tokens and removal:
            continue
        new_text.append(tokens)
    return new_text
def tokenize_corpus_basic ( text : List [ str ] , removal : bool = True ) -> List [ List [ str ] ] : tokenizer = BasicEnglishTokenizer ( ) new_text = [ ] for line in text : tokens = [ x for x in tokenizer . tokenize ( line ) if x not in string . punctuation ] if not tokens and removal : continue new_text . append ( tokens ) return new_text
Function to tokenize a list of strings (corpus).
Function to tokenize a list of strings ( corpus ) .
"""
    Function to tokenize a list of strings (corpus).

    Args:
        text (str): a list of strings to be converted into tokens.
        removal (bool): remove token list if it is empty
    Returns:
        a list of a list of tokens.
    """ # tokenize sentence
a list of a list of tokens.
a list of a list of tokens .
text
List[str]
a list of strings to be converted into tokens.
a list of strings to be converted into tokens .
removal
bool
remove token list if it is empty
remove token list if it is empty