paddlenlp/transformers/ernie_m/tokenizer.py
def clean_text(self, text):
        """Performs invalid character removal and whitespace cleanup on text."""
        text = text.replace(u"“", u'"')\
                .replace(u'”', u'"')\
                .replace(u'‘', "'")\
                .replace(u'’', u"'")\
                .replace(u'—', u'-')

        output = []
        for char in text:
            if _is_control(char):
                continue
            if _is_whitespace(char):
                output.append(" ")
            else:
                output.append(char)
        return "".join(output)
Performs invalid character removal and whitespace cleanup on text.
Performs invalid character removal and whitespace cleanup on text.
Performs invalid character removal and whitespace cleanup on text .
def clean_text(self, text):
        text = text.replace(u"“", u'"')\
                .replace(u'”', u'"')\
                .replace(u'‘', "'")\
                .replace(u'’', u"'")\
                .replace(u'—', u'-')
        output = []
        for char in text:
            if _is_control(char):
                continue
            if _is_whitespace(char):
                output.append(" ")
            else:
                output.append(char)
        return "".join(output)
def clean_text ( self , text ) : text = text . replace ( u"“",  u "')\ . replace ( u'”',  u "')\ . replace ( u'‘',  " ")\ . replace ( u'’',  u '")\ . replace ( u'—',  u -') output = [ ] for char in text : if _is_control ( char ) : continue if _is_whitespace ( char ) : output . append ( " " ) else : output . append ( char ) return "" . join ( output )
Performs invalid character removal and whitespace cleanup on text.
Performs invalid character removal and whitespace cleanup on text .
"""Performs invalid character removal and whitespace cleanup on text."""
self
text