src/python/txtai/tokenizer.py
def tokenize(text):
        """
        Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.

        Args:
            text: input text

        Returns:
            list of tokens
        """

        # Convert to all lowercase, split on whitespace, strip punctuation
        tokens = [token.strip(string.punctuation) for token in text.lower().split()]

        # Tokenize on alphanumeric strings.
        # Require strings to be at least 2 characters long.
        # Require at least 1 alpha character in string.
        return [token for token in tokens if re.match(r"^\d*[a-z][\-.0-9:_a-z]{1,}$", token) and token not in Tokenizer.STOP_WORDS]

        Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.

        Args:
            text: input text

        Returns:
            list of tokens
        
Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.
Tokenizes input text into a list of tokens . Filters tokens that match a specific pattern and removes stop words .
def tokenize(text):
        tokens = [token.strip(string.punctuation) for token in text.lower().split()]
        return [token for token in tokens if re.match(r"^\d*[a-z][\-.0-9:_a-z]{1,}$", token) and token not in Tokenizer.STOP_WORDS]
def tokenize ( text ) : tokens = [ token . strip ( string . punctuation ) for token in text . lower ( ) . split ( ) ] return [ token for token in tokens if re . match ( r"^\d*[a-z][\-.0-9:_a-z]{1,}$" , token ) and token not in Tokenizer . STOP_WORDS ]
Tokenizes input text into a list of tokens.
Tokenizes input text into a list of tokens .
"""
        Tokenizes input text into a list of tokens. Filters tokens that match a specific pattern and removes stop words.

        Args:
            text: input text

        Returns:
            list of tokens
        """ # Convert to all lowercase, split on whitespace, strip punctuation # Tokenize on alphanumeric strings. # Require strings to be at least 2 characters long. # Require at least 1 alpha character in string.
list of tokens
list of tokens
text
None