def tokenize(self, text):
    'Tokenizes a piece of text.'
    text = convert_to_unicode(text)
    text = self._clean_text(text)
    text = self._tokenize_chinese_chars(text)
    orig_tokens = whitespace_tokenize(text)
    split_tokens = []
    for token in orig_tokens:
        if self.do_lower_case:
            token = token.lower()
            token = self._run_strip_accents(token)
        split_tokens.extend(self._run_split_on_punc(token))
    output_tokens = whitespace_tokenize(' '.join(split_tokens))
    return output_tokens
Tokenizes a piece of text.
def tokenize(self, text):
    
    text = convert_to_unicode(text)
    text = self._clean_text(text)
    text = self._tokenize_chinese_chars(text)
    orig_tokens = whitespace_tokenize(text)
    split_tokens = []
    for token in orig_tokens:
        if self.do_lower_case:
            token = token.lower()
            token = self._run_strip_accents(token)
        split_tokens.extend(self._run_split_on_punc(token))
    output_tokens = whitespace_tokenize(' '.join(split_tokens))
    return output_tokens