'This function breaks the time string into lexical units (tokens), which
can be parsed by the parser. Lexical units are demarcated by changes in
the character set, so any continuous string of letters is considered
one unit, any continuous string of numbers is considered one unit.
The main complication arises from the fact that dots (\'.\') can be used
both as separators (e.g. "Sep.20.2009") or decimal points (e.g.
"4:30:21.447"). As such, it is necessary to read the full context of
any dot-separated strings before breaking it into tokens; as such, this
function maintains a "token stack", for when the ambiguous context
demands that multiple tokens be parsed at once.'

def get_token(self):

    if self.tokenstack:
        return self.tokenstack.pop(0)
    seenletters = False
    token = None
    state = None
    while (not self.eof):
        if self.charstack:
            nextchar = self.charstack.pop(0)
        else:
            nextchar = self.instream.read(1)
            while (nextchar == u'\x00'):
                nextchar = self.instream.read(1)
        if (not nextchar):
            self.eof = True
            break
        elif (not state):
            token = nextchar
            if self.isword(nextchar):
                state = u'a'
            elif self.isnum(nextchar):
                state = u'0'
            elif self.isspace(nextchar):
                token = u'    '
                break
            else:
                break
        elif (state == u'a'):
            seenletters = True
            if self.isword(nextchar):
                token += nextchar
            elif (nextchar == u'.'):
                token += nextchar
                state = u'a.'
            else:
                self.charstack.append(nextchar)
                break
        elif (state == u'0'):
            if self.isnum(nextchar):
                token += nextchar
            elif ((nextchar == u'.') or ((nextchar == u',') and (len(token) >= 2))):
                token += nextchar
                state = u'0.'
            else:
                self.charstack.append(nextchar)
                break
        elif (state == u'a.'):
            seenletters = True
            if ((nextchar == u'.') or self.isword(nextchar)):
                token += nextchar
            elif (self.isnum(nextchar) and (token[(-1)] == u'.')):
                token += nextchar
                state = u'0.'
            else:
                self.charstack.append(nextchar)
                break
        elif (state == u'0.'):
            if ((nextchar == u'.') or self.isnum(nextchar)):
                token += nextchar
            elif (self.isword(nextchar) and (token[(-1)] == u'.')):
                token += nextchar
                state = u'a.'
            else:
                self.charstack.append(nextchar)
                break
    if ((state in (u'a.', u'0.')) and (seenletters or (token.count(u'.') > 1) or (token[(-1)] in u'.,'))):
        l = self._split_decimal.split(token)
        token = l[0]
        for tok in l[1:]:
            if tok:
                self.tokenstack.append(tok)
    if ((state == u'0.') and (token.count(u'.') == 0)):
        token = token.replace(u',', u'.')
    return token
