'Since ERB doesn\'t allow "<%" and other tags inside of ruby
blocks we have to use a split approach here that fails for
that too.'

def get_tokens_unprocessed(self, text):

    tokens = self._block_re.split(text)
    tokens.reverse()
    state = idx = 0
    try:
        while True:
            if (state == 0):
                val = tokens.pop()
                (yield (idx, Other, val))
                idx += len(val)
                state = 1
            elif (state == 1):
                tag = tokens.pop()
                if (tag in ('<%%', '%%>')):
                    (yield (idx, Other, tag))
                    idx += 3
                    state = 0
                elif (tag == '<%#'):
                    (yield (idx, Comment.Preproc, tag))
                    val = tokens.pop()
                    (yield ((idx + 3), Comment, val))
                    idx += (3 + len(val))
                    state = 2
                elif (tag in ('<%', '<%=', '<%-')):
                    (yield (idx, Comment.Preproc, tag))
                    idx += len(tag)
                    data = tokens.pop()
                    r_idx = 0
                    for (r_idx, r_token, r_value) in self.ruby_lexer.get_tokens_unprocessed(data):
                        (yield ((r_idx + idx), r_token, r_value))
                    idx += len(data)
                    state = 2
                elif (tag in ('%>', '-%>')):
                    (yield (idx, Error, tag))
                    idx += len(tag)
                    state = 0
                else:
                    (yield (idx, Comment.Preproc, tag[0]))
                    r_idx = 0
                    for (r_idx, r_token, r_value) in self.ruby_lexer.get_tokens_unprocessed(tag[1:]):
                        (yield (((idx + 1) + r_idx), r_token, r_value))
                    idx += len(tag)
                    state = 0
            elif (state == 2):
                tag = tokens.pop()
                if (tag not in ('%>', '-%>')):
                    (yield (idx, Other, tag))
                else:
                    (yield (idx, Comment.Preproc, tag))
                idx += len(tag)
                state = 0
    except IndexError:
        return
