'Tokenizes an input string into a list of words.'

def _tokenize(self, item):

    tokenized = []
    for s in self._sentence_detector.tokenize(item):
        tokenized.extend(nltk.tokenize.word_tokenize(s))
    return tokenized
