paddlenlp/transformers/prophetnet/tokenizer.py
def _tokenize(self, text):
        """
        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List[str]: A list of string representing converted tokens.
        """
        no_split_token = set(self.unique_no_split_tokens)
        tokens = self.tokens_trie.split(text)
        for i, token in enumerate(tokens):
            if token in no_split_token:
                left = tokens[i - 1] if i > 0 else None
                right = tokens[i + 1] if i < len(tokens) - 1 else None
                # We strip left and right by default
                if right:
                    tokens[i + 1] = right.lstrip()
                if left:
                    tokens[i - 1] = left.rstrip()
        # ["This is something", "<special_token_1>", "else"]
        tokenized_text = []
        for token in tokens:
            # Need to skip eventual empty (fully stripped) tokens
            if not token:
                continue
            if token in no_split_token:
                tokenized_text.append(token)
            else:
                tokenized_text.extend(self._tokenize_function(token))
        # ["This", " is", " something", "<special_token_1>", "else"]
        return tokenized_text

        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List[str]: A list of string representing converted tokens.
        
Converts a string to a list of tokens.
Converts a string to a list of tokens .
def _tokenize(self, text):
        no_split_token = set(self.unique_no_split_tokens)
        tokens = self.tokens_trie.split(text)
        for i, token in enumerate(tokens):
            if token in no_split_token:
                left = tokens[i - 1] if i > 0 else None
                right = tokens[i + 1] if i < len(tokens) - 1 else None
                if right:
                    tokens[i + 1] = right.lstrip()
                if left:
                    tokens[i - 1] = left.rstrip()
        tokenized_text = []
        for token in tokens:
            if not token:
                continue
            if token in no_split_token:
                tokenized_text.append(token)
            else:
                tokenized_text.extend(self._tokenize_function(token))
        return tokenized_text
def _tokenize ( self , text ) : no_split_token = set ( self . unique_no_split_tokens ) tokens = self . tokens_trie . split ( text ) for i , token in enumerate ( tokens ) : if token in no_split_token : left = tokens [ i - 1 ] if i > 0 else None right = tokens [ i + 1 ] if i < len ( tokens ) - 1 else None if right : tokens [ i + 1 ] = right . lstrip ( ) if left : tokens [ i - 1 ] = left . rstrip ( ) tokenized_text = [ ] for token in tokens : if not token : continue if token in no_split_token : tokenized_text . append ( token ) else : tokenized_text . extend ( self . _tokenize_function ( token ) ) return tokenized_text
Converts a string to a list of tokens.
Converts a string to a list of tokens .
"""
        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List[str]: A list of string representing converted tokens.
        """ # We strip left and right by default
 # ["This is something", "<special_token_1>", "else"]
 # Need to skip eventual empty (fully stripped) tokens
 # ["This", " is", " something", "<special_token_1>", "else"]

A list of string representing converted tokens.
A list of string representing converted tokens .
List[str]
self
text
The text to be tokenized.
The text to be tokenized .