'Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.'

def get_tokens(self, text, unfiltered=False):

    if (not isinstance(text, unicode)):
        if (self.encoding == 'guess'):
            try:
                text = text.decode('utf-8')
                if text.startswith(u'\ufeff'):
                    text = text[len(u'\ufeff'):]
            except UnicodeDecodeError:
                text = text.decode('latin1')
        elif (self.encoding == 'chardet'):
            try:
                import chardet
            except ImportError:
                raise ImportError('To    enable    chardet    encoding    guessing,    please    install    the    chardet    library    from    http://chardet.feedparser.org/')
            enc = chardet.detect(text)
            text = text.decode(enc['encoding'])
        else:
            text = text.decode(self.encoding)
    text = text.replace('\r\n', '\n')
    text = text.replace('\r', '\n')
    if self.stripall:
        text = text.strip()
    elif self.stripnl:
        text = text.strip('\n')
    if (self.tabsize > 0):
        text = text.expandtabs(self.tabsize)
    if (self.ensurenl and (not text.endswith('\n'))):
        text += '\n'
    def streamer():
        for (i, t, v) in self.get_tokens_unprocessed(text):
            (yield (t, v))
    stream = streamer()
    if (not unfiltered):
        stream = apply_filters(stream, self.filters, self)
    return stream
