nlpmodels/utils/tokenizer.py
def tokenize(self, line: str) -> List[str]:
        """
        Function to tokenize a string within tokenizer.

        Args:
            line (str): the line to be tokenized
        Returns:
            a list of tokens
        """
        line = line.lower()
        for pattern_re, replaced_str in self._patterns_dict:
            # apply regex substitutes
            line = pattern_re.sub(replaced_str, line)
            # remove underscores
            line = line.replace('_', '')
        # remove all 1 letters
        line = line.split()
        line = [x for x in line if len(x) > 1]
        return line

        Function to tokenize a string within tokenizer.

        Args:
            line (str): the line to be tokenized
        Returns:
            a list of tokens
        
Function to tokenize a string within tokenizer.
Function to tokenize a string within tokenizer .
def tokenize(self, line: str) -> List[str]:
        line = line.lower()
        for pattern_re, replaced_str in self._patterns_dict:
            line = pattern_re.sub(replaced_str, line)
            line = line.replace('_', '')
        line = line.split()
        line = [x for x in line if len(x) > 1]
        return line
def tokenize ( self , line : str ) -> List [ str ] : line = line . lower ( ) for pattern_re , replaced_str in self . _patterns_dict : line = pattern_re . sub ( replaced_str , line ) line = line . replace ( '_' , '' ) line = line . split ( ) line = [ x for x in line if len ( x ) > 1 ] return line
Function to tokenize a string within tokenizer.
Function to tokenize a string within tokenizer .
"""
        Function to tokenize a string within tokenizer.

        Args:
            line (str): the line to be tokenized
        Returns:
            a list of tokens
        """ # apply regex substitutes # remove underscores # remove all 1 letters
a list of tokens
a list of tokens
self
line
str
the line to be tokenized
the line to be tokenized