FList<TextMateLexerState> (@NotNull CharSequence line, @NotNull Queue<Token> output, @NotNull FList<TextMateLexerState> states, int lineStartOffset, int linePosition, int lineByteOffset) { FList<TextMateLexerState> lastSuccessState = states; int lastSuccessStateOccursCount = 0; int lastMovedOffset = lineStartOffset; boolean matchBeginOfString = lineStartOffset == 0; int anchorByteOffset = -1; // makes sense only for a line, cannot be used across lines StringWithId string = new StringWithId(line); while (true) { final TextMateLexerState lexerState = states.getHead(); if (lexerState.syntaxRule.getStringAttribute(Constants.StringKey.WHILE) != null) { MatchData matchWhile = SyntaxMatchUtils.matchStringRegex(Constants.StringKey.WHILE, string, lineByteOffset, anchorByteOffset, matchBeginOfString, lexerState); if (!matchWhile.matched()) { closeScopeSelector(output, linePosition + lineStartOffset); closeScopeSelector(output, linePosition + lineStartOffset); states = states.getTail(); // this is happening at line start, none of the previous states couldn't be run on this line, so no need to update anchorByteOffset continue; } else { anchorByteOffset = matchWhile.byteOffset().end; } } break; } Set<TextMateLexerState> localStates = new HashSet<>(); while (true) { TextMateLexerState lastState = states.getHead(); SyntaxNodeDescriptor lastRule = lastState.syntaxRule; TextMateLexerState currentState = SyntaxMatchUtils.matchFirst(lastRule, string, lineByteOffset, anchorByteOffset, matchBeginOfString, TextMateWeigh.Priority.NORMAL, myCurrentScope); SyntaxNodeDescriptor currentRule = currentState.syntaxRule; MatchData currentMatch = currentState.matchData; int endPosition; MatchData endMatch = SyntaxMatchUtils.matchStringRegex(Constants.StringKey.END, string, lineByteOffset, anchorByteOffset, matchBeginOfString, lastState); if (endMatch.matched() && (!currentMatch.matched() || currentMatch.byteOffset().start >= endMatch.byteOffset().start || lastState.equals(currentState))) { TextMateLexerState poppedState = states.getHead(); if (poppedState.matchData.matched() && !poppedState.matchedEOL) { // if begin hasn't matched EOL, it was performed on the same line; we need to use its anchor anchorByteOffset = poppedState.matchData.byteOffset().end; } states = states.getTail(); TextMateRange endRange = endMatch.charRange(line, string.bytes); int startPosition = endPosition = endRange.start; closeScopeSelector(output, startPosition + lineStartOffset); // closing content scope if (lastRule.getCaptureRules(Constants.CaptureKey.END_CAPTURES) == null && lastRule.getCaptureRules(Constants.CaptureKey.CAPTURES) == null && lastRule.getCaptureRules(Constants.CaptureKey.BEGIN_CAPTURES) == null || parseCaptures(output, Constants.CaptureKey.END_CAPTURES, lastRule, endMatch, string, line, lineStartOffset, states) || parseCaptures(output, Constants.CaptureKey.CAPTURES, lastRule, endMatch, string, line, lineStartOffset, states)) { // move line position only if anything was captured or if there is nothing to capture at all endPosition = endRange.end; } closeScopeSelector(output, endPosition + lineStartOffset); // closing basic scope if (linePosition == endPosition && containsLexerState(localStates, poppedState) && poppedState.enterByteOffset == lineByteOffset) { addToken(output, line.length() + lineStartOffset); break; } localStates.remove(poppedState); } else if (currentMatch.matched()) { anchorByteOffset = currentMatch.byteOffset().end; TextMateRange currentRange = currentMatch.charRange(line, string.bytes); int startPosition = currentRange.start; endPosition = currentRange.end; if (currentRule.getStringAttribute(Constants.StringKey.BEGIN) != null) { states = states.prepend(currentState); CharSequence name = SyntaxMatchUtils.getStringAttribute(Constants.StringKey.NAME, currentRule, string, currentMatch); openScopeSelector(output, name, startPosition + lineStartOffset); parseCaptures(output, Constants.CaptureKey.BEGIN_CAPTURES, currentRule, currentMatch, string, line, lineStartOffset, states); parseCaptures(output, Constants.CaptureKey.CAPTURES, currentRule, currentMatch, string, line, lineStartOffset, states); CharSequence contentName = SyntaxMatchUtils.getStringAttribute(Constants.StringKey.CONTENT_NAME, currentRule, string, currentMatch); openScopeSelector(output, contentName, endPosition + lineStartOffset); } else if (currentRule.getStringAttribute(Constants.StringKey.MATCH) != null) { CharSequence name = SyntaxMatchUtils.getStringAttribute(Constants.StringKey.NAME, currentRule, string, currentMatch); openScopeSelector(output, name, startPosition + lineStartOffset); parseCaptures(output, Constants.CaptureKey.CAPTURES, currentRule, currentMatch, string, line, lineStartOffset, states); closeScopeSelector(output, endPosition + lineStartOffset); } if (linePosition == endPosition && containsLexerState(localStates, currentState)) { addToken(output, line.length() + lineStartOffset); break; } localStates.add(currentState); } else { addToken(output, line.length() + lineStartOffset); break; } // global looping protection if (lastMovedOffset < myCurrentOffset) { lastSuccessState = states; lastSuccessStateOccursCount = 0; lastMovedOffset = myCurrentOffset; } else if (lastSuccessState.equals(states)) { if (lastSuccessStateOccursCount > MAX_LOOPS_COUNT) { addToken(output, line.length() + lineStartOffset); break; } lastSuccessStateOccursCount++; } if (linePosition != endPosition) { linePosition = endPosition; lineByteOffset = RegexUtil.byteOffsetByCharOffset(line, linePosition); } if (myCheckCancelledCallback != null) { myCheckCancelledCallback.run(); } } return states; }