addToken(info,pos - lastOffset,
context = (LineContext) context.parent;  if ((context.inRule.action & EXCLUDE_MATCH) == EXCLUDE_MATCH) { addToken(info,pattern.count,
	protected byte markTokensImpl(byte token, Segment line, int idx,		LineInfo info)	{		LineContext lastContext =			(idx == 0 || lineInfo[idx - 1].obj == null)			? new LineContext(null, getRuleSet("MAIN"))			: ((LineContext) lineInfo[idx - 1].obj);		try		{			context = (LineContext) lastContext.clone();		}		catch (CloneNotSupportedException cnse)		{			context = new LineContext();		}		lastOffset = lastKeyword = line.offset;		lineLength = line.count + line.offset;		int terminateChar = context.rules.getTerminateChar();		int searchLimit = (terminateChar >= 0 && terminateChar < line.count)			? line.offset + terminateChar : lineLength;		escaped = false;		boolean b;		ParserRule tempRule;		Enumeration rule_enum;		for(pos = line.offset; pos < searchLimit; pos++)		{			if (context.parent != null)			{				LineContext tempContext = context;				context = context.parent;				pattern.array = context.inRule.searchChars;				pattern.count = context.inRule.sequenceLengths[1];				pattern.offset = context.inRule.sequenceLengths[0];				b = handleRule(info,line, context.inRule);				context = tempContext;				if (!b)				{					if (pos != lastOffset)					{						if (context.inRule == null)						{							if(context.rules.getKeywords() != null)							{								markKeyword(info,line, lastKeyword, pos);							}							addToken(info,pos - lastOffset,								getActionToken(context.rules.getDefault()));						}						else if ((context.inRule.action & (NO_LINE_BREAK | NO_WORD_BREAK)) == 0)						{							addToken(info,pos - lastOffset,								getActionToken(context.inRule.action));						}						else						{							addToken(info,pos - lastOffset, Token.INVALID);						}					}					context = (LineContext) context.parent;					if ((context.inRule.action & EXCLUDE_MATCH) == EXCLUDE_MATCH)					{						addToken(info,pattern.count,							getActionToken(context.rules.getDefault()));					}					else					{						addToken(info,pattern.count, getActionToken(context.inRule.action));					}					context.inRule = null;					lastKeyword = lastOffset = pos + pattern.count;					pos += (pattern.count - 1); // move pos to last character of match sequence					continue;				}			}			if (context.rules.getEscapeRule() != null)			{				// assign tempPattern to mutable "buffer" pattern				Segment tempPattern = pattern;				// swap in the escape pattern				pattern = context.rules.getEscapePattern();				b = handleRule(info,line, context.rules.getEscapeRule());				// swap back the buffer pattern				pattern = tempPattern;				if (!b) continue;			}			if (context.inRule != null && (context.inRule.action & SOFT_SPAN) == 0)			{				// we are in a hard span rule, so see if context.inRule matches				pattern.array = context.inRule.searchChars;				pattern.count = context.inRule.sequenceLengths[1];				pattern.offset = context.inRule.sequenceLengths[0];				if (handleRule(info,line, context.inRule) && escaped)					escaped = false;			}			else			{				// otherwise, if this isn't a hard span, check every rule				rule_enum = context.rules.getRules();				while (rule_enum.hasMoreElements())				{					tempRule = (ParserRule) rule_enum.nextElement();					pattern.array = tempRule.searchChars;					if (context.inRule == tempRule && (tempRule.action & SPAN) == SPAN)					{						pattern.count = tempRule.sequenceLengths[1];						pattern.offset = tempRule.sequenceLengths[0];					}					else					{						pattern.count = tempRule.sequenceLengths[0];						pattern.offset = 0;					}					// stop checking rules if there was a match and go to next pos					if (!handleRule(info,line, tempRule)) break;				}			}		}		// check for keywords at the line's end		if(context.rules.getKeywords() != null && context.inRule == null)		{			markKeyword(info,line, lastKeyword, lineLength);		}		// mark all remaining characters		if(lastOffset != lineLength)		{			if (context.inRule == null)			{				addToken(info,lineLength - lastOffset,					getActionToken(context.rules.getDefault()));			}			else if (				(context.inRule.action & SPAN) == SPAN &&				(context.inRule.action & (NO_LINE_BREAK | NO_WORD_BREAK)) != 0			)			{				addToken(info,lineLength - lastOffset,Token.INVALID);				context.inRule = null;			}			else			{				addToken(info,lineLength - lastOffset, getActionToken(context.inRule.action));				if((context.inRule.action & MARK_FOLLOWING) == MARK_FOLLOWING)				{					context.inRule = null;				}			}		}		lineInfo[idx].obj = context;		return getActionToken((context.inRule == null) ? context.rules.getDefault() :			context.inRule.action);	}