paddlenlp/transformers/t5/tokenizer.py
def clean_up_tokenization(out_string):
        """
        Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.

        Args:
            out_string (str): The text to clean up.

        Returns:
            str: The cleaned-up string.
        """
        out_string = (out_string.replace(" .", ".").replace(" ?", "?")
                      .replace(" !", "!").replace(" ,", ",").replace(" ' ", "'")
                      .replace(" n't", "n't").replace(" 'm", "'m")
                      .replace(" 's", "'s").replace(" 've", "'ve")
                      .replace(" 're", "'re"))
        return out_string

        Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.

        Args:
            out_string (str): The text to clean up.

        Returns:
            str: The cleaned-up string.
        
Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.
Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms .
def clean_up_tokenization(out_string):
        out_string = (out_string.replace(" .", ".").replace(" ?", "?")
                      .replace(" !", "!").replace(" ,", ",").replace(" ' ", "'")
                      .replace(" n't", "n't").replace(" 'm", "'m")
                      .replace(" 's", "'s").replace(" 've", "'ve")
                      .replace(" 're", "'re"))
        return out_string
def clean_up_tokenization ( out_string ) : out_string = ( out_string . replace ( " ." , "." ) . replace ( " ?" , "?" ) . replace ( " !" , "!" ) . replace ( " ," , "," ) . replace ( " ' " , "'" ) . replace ( " n't" , "n't" ) . replace ( " 'm" , "'m" ) . replace ( " 's" , "'s" ) . replace ( " 've" , "'ve" ) . replace ( " 're" , "'re" ) ) return out_string
Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.
Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms .
"""
        Clean up a list of simple English tokenization artifacts like spaces before punctuations and abbreviated forms.

        Args:
            out_string (str): The text to clean up.

        Returns:
            str: The cleaned-up string.
        """
The cleaned-up string.
The cleaned - up string .
str
out_string
The text to clean up.
The text to clean up .