 
   createPattern: (options) -> new Pattern(this, @registry, options)
 
-  tokenizeLine: (line, ruleStack, firstLine=false) ->
-    if ruleStack?
-      ruleStack = ruleStack.slice()
-    else
-      ruleStack = [@getInitialRule()]
-    originalRuleStack = ruleStack
-
-    tokens = []
-    position = 0
-
-    loop
-      scopes = @scopesFromStack(ruleStack)
-      previousRuleStackLength = ruleStack.length
-      previousPosition = position
-
-      if tokens.length >= @getMaxTokensPerLine() - 1
-        token = @createToken(line[position..], scopes)
-        tokens.push token
-        ruleStack = originalRuleStack
-        break
-
-      break if position == line.length + 1 # include trailing newline position
-
-      if match = _.last(ruleStack).getNextTokens(ruleStack, line, position, firstLine)
-        { nextTokens, tokensStartPosition, tokensEndPosition } = match
-        if position < tokensStartPosition # unmatched text before next tokens
-          tokens.push(@createToken(line[position...tokensStartPosition], scopes))
-
-        tokens.push(nextTokens...)
-        position = tokensEndPosition
-
-        # break if at end of the line and no tokens were in the match and no rule was pushed to the stack
-        if position is line.length and nextTokens.length is 0 and ruleStack.length is previousRuleStackLength
-          if tokens.length is 0 # push filler token for unmatched text
-            tokens.push(@createToken(line[position...line.length], scopes))
-          break
-
-      else # push filler token for unmatched text at end of line
-        if position < line.length or line.length == 0
-          tokens.push(@createToken(line[position...line.length], scopes))
-        break
-
-      if position == previousPosition
-        if ruleStack.length == previousRuleStackLength
-          console.error("Popping rule because it loops at column #{position} of line '#{line}'", _.clone(ruleStack))
-          ruleStack.pop()
-        else if ruleStack.length > previousRuleStackLength # Stack size increased with zero length match
-          [penultimateRule, lastRule] = ruleStack[-2..]
-
-          # Same exact rule was pushed but position wasn't advanced
-          if lastRule? and lastRule == penultimateRule
-            popStack = true
-
-          # Rule with same scope name as previous rule was pushed but position wasn't advanced
-          if lastRule?.scopeName? and penultimateRule.scopeName == lastRule.scopeName
-            popStack = true
-
-          if popStack
-            ruleStack.pop()
-            tokens.push(@createToken(line[position...line.length], scopes))
-            break
-
-    rule.clearAnchorPosition() for rule in ruleStack
-    {tokens, ruleStack}
-
-  tokenizeLines: (text) ->
-    lines = text.split('\n')
-    ruleStack = null
-    for line, lineNumber in lines
-      {tokens, ruleStack} = @tokenizeLine(line, ruleStack, lineNumber is 0)
-      tokens
-
   getMaxTokensPerLine: ->
     @maxTokensPerLine
 


    @fileTypes ?= []
    @includedGrammarScopes = []

  activate: ->
    @registry.addGrammar(this)


pathSplitRegex = new RegExp("[#{_.escapeRegExp(path.sep)}.]")

# Public: Grammar that tokenizes lines of text.
#
# This class should not be instantiated directly but instead obtained from
# a {GrammarRegistry} by calling {GrammarRegistry.loadGrammar}.
module.exports =
class Grammar
  Emitter.includeInto(this)
