info = lineInfo[i]; if(info.tokensValid) { prev = info; continue; }  Element lineElement = map.getElement(i); int lineStart = lineElement.getStartOffset(); try { buffer.getText(lineStart,lineElement.getEndOffset() - lineStart - 1,line); } catch(BadLocationException e) { Log.log(Log.ERROR,this,e); }   info.lastToken = null;  ParserRule oldRule = info.context.inRule; LineContext oldParent = info.context.parent; markTokensImpl(prev,info); ParserRule newRule = info.context.inRule; LineContext newParent = info.context.parent;  info.tokensValid = true;  if(i != lastTokenizedLine) { nextLineRequested = false; lastTokenizedLine = i; }  nextLineRequested |= (oldRule != newRule || oldParent != newParent);  addToken(info,0,Token.END);  prev = info;
if (context.parent != null) { tempContext = context;  context = context.parent;  pattern.array = context.inRule.searchChars; pattern.count = context.inRule.sequenceLengths[1]; pattern.offset = context.inRule.sequenceLengths[0];  b = handleRule(info, line, context.inRule);  context = tempContext;  if (!b) { if (escaped) { escaped = false; } else { if (pos != lastOffset) { if (context.inRule == null) { markKeyword(info,line,lastKeyword,pos);  info.addToken(pos - lastOffset, context.rules.getDefault()); } else if ((context.inRule.action & (NO_LINE_BREAK | NO_WORD_BREAK)) == 0) { info.addToken(pos - lastOffset, context.inRule.token); } else { info.addToken(pos - lastOffset, Token.INVALID); } }  context = (LineContext) context.parent;  if ((context.inRule.action & EXCLUDE_MATCH) == EXCLUDE_MATCH) { info.addToken(pattern.count, context.rules.getDefault()); } else { info.addToken(pattern.count,context.inRule.token); }  context.inRule = null;  lastKeyword = lastOffset = pos + pattern.count; }  pos += (pattern.count - 1);  continue; } }   if ((rule = context.rules.getEscapeRule()) != null) {  tempPattern = pattern;   pattern = context.rules.getEscapePattern();  tempEscaped = escaped;  b = handleRule(info, line, rule);   pattern = tempPattern;  if (!b) { if (tempEscaped) escaped = false; continue; } }   rule = context.inRule; if(rule != null && (rule.action & SPAN) == SPAN) { pattern.array = rule.searchChars; pattern.count = rule.sequenceLengths[1]; pattern.offset = rule.sequenceLengths[0];     if (!handleRule(info,line,rule) || (rule.action & SOFT_SPAN) == 0) { escaped = false; continue; } }   rule = context.rules.getRules(line.array[pos]); while(rule != null) { pattern.array = rule.searchChars;  if (context.inRule == rule && (rule.action & SPAN) == SPAN) { pattern.count = rule.sequenceLengths[1]; pattern.offset = rule.sequenceLengths[0]; } else { pattern.count = rule.sequenceLengths[0]; pattern.offset = 0; }   if (!handleRule(info,line,rule)) break;  rule = rule.next; }  escaped = false;
	public LineInfo markTokens(Buffer buffer, int lineIndex)	{		LineInfo info = lineInfo[lineIndex];		/* If cached tokens are valid, return 'em */		if(info.tokensValid)			return info;		//long _start = System.currentTimeMillis();		/*		 * Else, go up to 100 lines back, looking for a line with		 * cached tokens. Tokenize from that line to this line.		 */		int start = Math.max(0,lineIndex - 100) - 1;		int end = Math.max(0,lineIndex - 100);		for(int i = lineIndex - 1; i > end; i--)		{			if(lineInfo[i].tokensValid)			{				start = i;				break;			}		}		LineInfo prev;		if(start == -1)			prev = null;		else			prev = lineInfo[start];		//System.err.println("i=" + lineIndex + ",start=" + start);		Element map = buffer.getDefaultRootElement();		for(int i = start + 1; i <= lineIndex; i++)		{			info = lineInfo[i];			if(info.tokensValid)			{				prev = info;				continue;			}			Element lineElement = map.getElement(i);			int lineStart = lineElement.getStartOffset();			try			{				buffer.getText(lineStart,lineElement.getEndOffset()					- lineStart - 1,line);			}			catch(BadLocationException e)			{				Log.log(Log.ERROR,this,e);			}			/* Prepare for tokenization */			info.lastToken = null;			ParserRule oldRule = info.context.inRule;			LineContext oldParent = info.context.parent;			markTokensImpl(prev,info);			ParserRule newRule = info.context.inRule;			LineContext newParent = info.context.parent;			info.tokensValid = true;			if(i != lastTokenizedLine)			{				nextLineRequested = false;				lastTokenizedLine = i;			}			nextLineRequested |= (oldRule != newRule || oldParent != newParent);			addToken(info,0,Token.END);			prev = info;		}		if(nextLineRequested && length - lineIndex > 1)		{			linesChanged(lineIndex + 1,length - lineIndex - 1);		}		//System.err.println(System.currentTimeMillis() - _start);		return info;	}