paddlenlp/transformers/t5/tokenizer.py
def convert_tokens_to_string(self, tokens):
        """Converts a sequence of tokens (string) in a single string."""
        current_sub_tokens = []
        out_string = ""
        for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                out_string += (self.sp_model.decode_pieces(current_sub_tokens) +
                               token + " ")
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
        out_string += self.sp_model.decode_pieces(current_sub_tokens)
        return out_string.strip()
Converts a sequence of tokens (string) in a single string.
Converts a sequence of tokens (string) in a single string.
Converts a sequence of tokens ( string ) in a single string .
def convert_tokens_to_string(self, tokens):
        current_sub_tokens = []
        out_string = ""
        for token in tokens:
            if token in self.all_special_tokens:
                out_string += (self.sp_model.decode_pieces(current_sub_tokens) +
                               token + " ")
                current_sub_tokens = []
            else:
                current_sub_tokens.append(token)
        out_string += self.sp_model.decode_pieces(current_sub_tokens)
        return out_string.strip()
def convert_tokens_to_string ( self , tokens ) : current_sub_tokens = [ ] out_string = "" for token in tokens : if token in self . all_special_tokens : out_string += ( self . sp_model . decode_pieces ( current_sub_tokens ) + token + " " ) current_sub_tokens = [ ] else : current_sub_tokens . append ( token ) out_string += self . sp_model . decode_pieces ( current_sub_tokens ) return out_string . strip ( )
Converts a sequence of tokens (string) in a single string.
Converts a sequence of tokens ( string ) in a single string .
"""Converts a sequence of tokens (string) in a single string.""" # make sure that special tokens are not decoded using sentencepiece model
self
tokens