'Tokenize a string of text.
>>> parser = CoreNLPParser(url=\'http://localhost:9000\')
>>> text = \'Good muffins cost $3.88\nin New York.  Please buy me\ntwo of them.\nThanks.\'
>>> list(parser.tokenize(text))
[\'Good\', \'muffins\', \'cost\', \'$\', \'3.88\', \'in\', \'New\', \'York\', \'.\', \'Please\', \'buy\', \'me\', \'two\', \'of\', \'them\', \'.\', \'Thanks\', \'.\']
>>> s = "The colour of the wall is blue."
>>> list(
...     parser.tokenize(
...         \'The colour of the wall is blue.\',
...             properties={\'tokenize.options\': \'americanize=true\'},
[\'The\', \'color\', \'of\', \'the\', \'wall\', \'is\', \'blue\', \'.\']'

def tokenize(self, text, properties=None):

    default_properties = {u'annotators': u'tokenize,ssplit'}
    default_properties.update((properties or {}))
    result = self.api_call(text, properties=default_properties)
    for sentence in result[u'sentences']:
        for token in sentence[u'tokens']:
            (yield (token[u'originalText'] or token[u'word']))
