'This is where the magic happens.
We do our usually processing through the states and when we have a token
to return we yield the token which pauses processing until the next token
is requested.'

def __iter__(self):

    self.tokenQueue = deque([])
    while self.state():
        while self.stream.errors:
            (yield {u'type': tokenTypes[u'ParseError'], u'data': self.stream.errors.pop(0)})
        while self.tokenQueue:
            (yield self.tokenQueue.popleft())
