 
   constructor: ({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) ->
     @startBufferColumn ?= 0
+
+    @specialTokens = {}
+
+    @subdivideTokens()
+
     # @tokens = @breakOutAtomicTokens(tokens)
     @bufferDelta = @buildBufferDelta()
     @softWrapIndentationTokens = @getSoftWrapIndentationTokens()

    if @invisibles
      @substituteInvisibleCharacters()
      @buildEndOfLineInvisibles() if @lineEnding?

  Object.defineProperty @prototype, 'tokens', get: ->
    tokens = atom.grammars.decodeContent(@text, @tags, @parentScopes.slice())

_ = require 'underscore-plus'
{isPairedCharacter} = require './text-utils'
Token = require './token'

SoftTab = Symbol('SoftTab')

NonWhitespaceRegex = /\S/
LeadingWhitespaceRegex = /^\s*/
