 _ = require 'underscore-plus'
 {isPairedCharacter} = require './text-utils'
 Token = require './token'
+
+SoftTab = Symbol('SoftTab')
 
 NonWhitespaceRegex = /\S/
 LeadingWhitespaceRegex = /^\s*/

    if @invisibles
      @substituteInvisibleCharacters()
      @buildEndOfLineInvisibles() if @lineEnding?

  Object.defineProperty @prototype, 'tokens', get: ->
    tokens = atom.grammars.decodeContent(@text, @tags, @parentScopes.slice())


  constructor: ({@parentScopes, @text, @tags, @lineEnding, @ruleStack, @startBufferColumn, @fold, @tabLength, @indentLevel, @invisibles}) ->
    @startBufferColumn ?= 0

    @specialTokens = {}

    @subdivideTokens()

    # @tokens = @breakOutAtomicTokens(tokens)
    @bufferDelta = @buildBufferDelta()
    @softWrapIndentationTokens = @getSoftWrapIndentationTokens()
