'Return a function that splits a string into a sequence of tokens'

def build_tokenizer(self):

    if (self.tokenizer is not None):
        return self.tokenizer
    token_pattern = re.compile(self.token_pattern)
    return (lambda doc: token_pattern.findall(doc))
