'Return a callable that handles preprocessing and tokenization'

def build_analyzer(self):

    if callable(self.analyzer):
        return self.analyzer
    preprocess = self.build_preprocessor()
    if (self.analyzer == u'char'):
        return (lambda doc: self._char_ngrams(preprocess(self.decode(doc))))
    elif (self.analyzer == u'char_wb'):
        return (lambda doc: self._char_wb_ngrams(preprocess(self.decode(doc))))
    elif (self.analyzer == u'word'):
        stop_words = self.get_stop_words()
        tokenize = self.build_tokenizer()
        return (lambda doc: self._word_ngrams(tokenize(preprocess(self.decode(doc))), stop_words))
    else:
        raise ValueError((u'%s    is    not    a    valid    tokenization    scheme/analyzer' % self.analyzer))
