            @commentToken()    or
            @whitespaceToken() or
            @lineToken()       or
-           @heredocToken()    or
            @stringToken()     or
            @numberToken()     or
            @regexToken()      or


MULTI_DENT = /^(?:\n[^\n\S]*)+/

SIMPLESTR  = /^'[^\\']*(?:\\[\s\S][^\\']*)*'/

JSTOKEN    = /^`[^\\`]*(?:\\.[^\\`]*)*`/

# Regex-matching-regexes.
REGEX = /// ^
  (/ (?! [\s=] )   # disallow leading whitespace or equals signs
  [^ [ / \n \\ ]*  # every other thing
  (?:
    (?: \\[\s\S]   # anything escaped
      | \[         # character class
           [^ \] \n \\ ]*
           (?: \\[\s\S] [^ \] \n \\ ]* )*
         ]
    ) [^ [ / \n \\ ]*
  )*
  /) ([imgy]{0,4}) (?!\w)
///

HEREGEX      = /// ^ /{3} ((?:\\?[\s\S])+?) /{3} ([imgy]{0,4}) (?!\w) ///

HEREGEX_OMIT = ///
    ((?:\\\\)+)     # consume (and preserve) an even number of backslashes

  closeIndentation: ->
    @outdentToken @indent

  # Match the contents of a delimited token and expand variables and expressions
  # inside it using Ruby-like notation for substitution of arbitrary
  # expressions.
  #
  #     "Hello #{name.capitalize()}."
  #
  # If it encounters an interpolation, this method will recursively create a new
  # Lexer and tokenize until the `{` of `#{` is balanced with a `}`.
  #
  #  - `str` is the start of the token contents (with the starting delimiter
  #    stripped off.)
  #  - `regex` matches the contents of a token (but not `end`, and not `#{` if
  #    interpolations are desired).
  #  - `end` is the terminator of the token.
  #  - `offsetInChunk` is the start of the interpolated string in the current
  #    chunk, including the starting delimiter.
  #
  # Examples of delimiters are `'`, `"`, `'''`, `"""` and `///`.
  #
  # This method allows us to have strings within interpolations within strings,
  # ad infinitum.
  matchWithInterpolations: (str, regex, end, offsetInChunk) ->
    tokens = []
    loop
      [strPart] = regex.exec str

      # Push a fake 'NEOSTRING' token, which will get turned into a real string later.
      tokens.push @makeToken 'NEOSTRING', strPart, offsetInChunk

      str = str[strPart.length..]
      offsetInChunk += strPart.length

      break unless str[...2] is '#{'

      # The `1`s are to remove the `#` in `#{`.
      [line, column] = @getLineAndColumnFromChunk offsetInChunk + 1
      {tokens: nested, index} =
        new Lexer().tokenize str[1..], line: line, column: column, untilBalanced: on
      # Skip the trailing `}`.
      index += 1

      # Remove leading and trailing `{` and `}`.
      nested.shift()
      nested.pop()

      # Remove leading 'TERMINATOR' (if any).
      nested.shift() if nested[0]?[0] is 'TERMINATOR'

      if nested.length > 1
        nested.unshift @makeToken '(', '(', offsetInChunk + 1, 0
        nested.push    @makeToken ')', ')', offsetInChunk + 1 + index, 0
      # Push a fake 'TOKENS' token, which will get turned into real tokens later.
      tokens.push ['TOKENS', nested]

      str = str[index..]
      offsetInChunk += index

    unless str[...end.length] is end
      @error "missing #{end}"

    {tokens, index: offsetInChunk + end.length}

  # Merge the array `tokens` of the fake token types 'TOKENS' and 'NEOSTRING'
  # (as returned by `matchWithInterpolations`) into the token stream. The value
  # of 'NEOSTRING's are converted using `fn` and turned into strings using
  # `quote` first.  The tokens are wrapped in parentheses if needed, using
  # `start` and `end` for their location data.
  mergeInterpolationTokens: (tokens, {quote, start, end}, fn) ->
    if interpolated = tokens.length > 1
      errorToken = @makeToken '', 'interpolation', start + tokens[0][1].length, 2
      @token '(', '(', 0, 0, errorToken

    firstIndex = @tokens.length
    for token, i in tokens
      [tag, value] = token
      switch tag
        when 'TOKENS'
          # Optimize out empty interpolations.
          continue if value.length is 0
          # Push all the tokens in the fake 'TOKENS' token. These already have
          # sane location data.
          locationToken = value[0]
          tokensToPush = value
        when 'NEOSTRING'
          # Convert 'NEOSTRING' into 'STRING'.
          converted = fn token[1], i
          # Optimize out empty strings. We ensure that the tokens stream always
          # starts with a string token, though, to make sure that the result
          # really is a string.
          if converted.length is 0
            if i is 0
              firstEmptyStringIndex = @tokens.length
            else
              continue
          # However, there is one case where we can optimize away a starting
          # empty string.
          if i is 2 and firstEmptyStringIndex?
            @tokens.splice firstEmptyStringIndex, 2 # Remove empty string and the plus.
          token[0] = 'STRING'
          token[1] = @makeString converted, quote
          locationToken = token
          tokensToPush = [token]
      if @tokens.length > firstIndex
        # Create a 0-length "+" token.
        plusToken = @token '+', '+'
        plusToken[2] =
          first_line:   locationToken[2].first_line
          first_column: locationToken[2].first_column
          last_line:    locationToken[2].first_line
          last_column:  locationToken[2].first_column
      @tokens.push tokensToPush...

    if interpolated
      rparen = @token ')', ')', end, 0
      rparen.stringEnd = true

  # Pairs up a closing token, ensuring that all listed pairs of tokens are
  # correctly balanced throughout the course of the token stream.
