paddlenlp/transformers/ctrl/tokenizer.py
def convert_tokens_to_ids(self, tokens):
        """
        Converts a single token or a sequence of tokens to an index or a
        sequence of indices using the vocab.

        Args:
            tokens (str|List[str]|tuple(str)):
                A single token or a sequence of tokens.

        Returns:
            int|List[int]: The converted token id or token ids.

        Example:
            .. code-block::

                from paddlenlp.transformers import CTRLTokenizer

                tokenizer = CTRLTokenizer.from_pretrained('crtl')
                print(tokenizer.convert_tokens_to_ids(['Welcome', 'to', 'use', 'Padd@@', 'le@@', 'Padd@@', 'le', 'and', 'Padd@@', 'le@@', 'N@@', 'LP']))
                # [41116, 3, 191, 40324, 1162, 40324, 992, 2, 40324, 1162, 633, 11135]

        """
        ids = []
        if isinstance(tokens, str):
            return self._convert_token_to_id(tokens)
        for token in tokens:
            ids.append(self._convert_token_to_id(token))
        if len(ids) > self.max_len:
            logger.warning(
                "Token indices sequence length is longer than the specified maximum "
                " sequence length for this CTRL model ({} > {}). Running this"
                " sequence through the model will result in indexing errors".
                format(len(ids), self.max_len))
        return ids

        Converts a single token or a sequence of tokens to an index or a
        sequence of indices using the vocab.

        Args:
            tokens (str|List[str]|tuple(str)):
                A single token or a sequence of tokens.

        Returns:
            int|List[int]: The converted token id or token ids.

        Example:
            .. code-block::

                from paddlenlp.transformers import CTRLTokenizer

                tokenizer = CTRLTokenizer.from_pretrained('crtl')
                print(tokenizer.convert_tokens_to_ids(['Welcome', 'to', 'use', 'Padd@@', 'le@@', 'Padd@@', 'le', 'and', 'Padd@@', 'le@@', 'N@@', 'LP']))
                # [41116, 3, 191, 40324, 1162, 40324, 992, 2, 40324, 1162, 633, 11135]

        
Converts a single token or a sequence of tokens to an index or a
sequence of indices using the vocab.
Converts a single token or a sequence of tokens to an index or a sequence of indices using the vocab .
def convert_tokens_to_ids(self, tokens):
        ids = []
        if isinstance(tokens, str):
            return self._convert_token_to_id(tokens)
        for token in tokens:
            ids.append(self._convert_token_to_id(token))
        if len(ids) > self.max_len:
            logger.warning(
                "Token indices sequence length is longer than the specified maximum "
                " sequence length for this CTRL model ({} > {}). Running this"
                " sequence through the model will result in indexing errors".
                format(len(ids), self.max_len))
        return ids
def convert_tokens_to_ids ( self , tokens ) : ids = [ ] if isinstance ( tokens , str ) : return self . _convert_token_to_id ( tokens ) for token in tokens : ids . append ( self . _convert_token_to_id ( token ) ) if len ( ids ) > self . max_len : logger . warning ( "Token indices sequence length is longer than the specified maximum " " sequence length for this CTRL model ({} > {}). Running this" " sequence through the model will result in indexing errors" . format ( len ( ids ) , self . max_len ) ) return ids
Converts a single token or a sequence of tokens to an index or a
sequence of indices using the vocab.
Converts a single token or a sequence of tokens to an index or a sequence of indices using the vocab .
"""
        Converts a single token or a sequence of tokens to an index or a
        sequence of indices using the vocab.

        Args:
            tokens (str|List[str]|tuple(str)):
                A single token or a sequence of tokens.

        Returns:
            int|List[int]: The converted token id or token ids.

        Example:
            .. code-block::

                from paddlenlp.transformers import CTRLTokenizer

                tokenizer = CTRLTokenizer.from_pretrained('crtl')
                print(tokenizer.convert_tokens_to_ids(['Welcome', 'to', 'use', 'Padd@@', 'le@@', 'Padd@@', 'le', 'and', 'Padd@@', 'le@@', 'N@@', 'LP']))
                # [41116, 3, 191, 40324, 1162, 40324, 992, 2, 40324, 1162, 633, 11135]

        """
The converted token id or token ids.
The converted token id or token ids .
int|List[int]
self
tokens
A single token or a sequence of tokens.
A single token or a sequence of tokens .
examples
code-block:.

from paddlenlp.transformers import CTRLTokenizer


code - block : . from paddlenlp . transformers import CTRLTokenizer