paddlenlp/transformers/albert/tokenizer.py
def tokenize(self, text):
        """
        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List(str): A list of string representing converted tokens.

        Examples:
            .. code-block::

                from paddlenlp.transformers import RobertaTokenizer

                tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
                tokens = tokenizer.tokenize('He was a puppeteer')

        """

        return self.tokenizer.tokenize(text)

        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List(str): A list of string representing converted tokens.

        Examples:
            .. code-block::

                from paddlenlp.transformers import RobertaTokenizer

                tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
                tokens = tokenizer.tokenize('He was a puppeteer')

        
Converts a string to a list of tokens.
Converts a string to a list of tokens .
def tokenize(self, text):
        return self.tokenizer.tokenize(text)
def tokenize ( self , text ) : return self . tokenizer . tokenize ( text )
Converts a string to a list of tokens.
Converts a string to a list of tokens .
"""
        Converts a string to a list of tokens.

        Args:
            text (str): The text to be tokenized.

        Returns:
            List(str): A list of string representing converted tokens.

        Examples:
            .. code-block::

                from paddlenlp.transformers import RobertaTokenizer

                tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
                tokens = tokenizer.tokenize('He was a puppeteer')

        """
A list of string representing converted tokens.
A list of string representing converted tokens .
List(str)
self
text
The text to be tokenized.
The text to be tokenized .
examples
code-block:.

from paddlenlp.transformers import RobertaTokenizer

tokenizer = RobertaTokenizer.from_pretrained('roberta-wwm-ext')
tokens = tokenizer.tokenize('He was a puppeteer')
code - block : . from paddlenlp . transformers import RobertaTokenizer tokenizer = RobertaTokenizer . from_pretrained ( ' roberta - wwm - ext ' ) tokens = tokenizer . tokenize ( ' He was a puppeteer ' )