class_docstr: Class that implements a lexer for a given environment. Automatically
  created by the environment class, usually you don't have to do that.
  
  Note that the lexer is not automatically bound to an environment.
  Multiple environments can share the same lexer.
class_docstr_tok: ["Class","that","implements","a","lexer","for","a","given","environment",".","Automatically","created","by","the","environment","class",",","usually","you","do","n't","have","to","do","that",".","Note","that","the","lexer","is","not","automatically","bound","to","an","environment",".","Multiple","environments","can","share","the","same","lexer","."]
code: def __init__(self, environment):
      # shortcuts
      e = re.escape
  
      def c(x):
          return re.compile(x, re.M | re.S)
  
      # lexing rules for tags
      tag_rules = [
          (whitespace_re, TOKEN_WHITESPACE, None),
          (float_re, TOKEN_FLOAT, None),
          (integer_re, TOKEN_INTEGER, None),
          (name_re, TOKEN_NAME, None),
          (string_re, TOKEN_STRING, None),
          (operator_re, TOKEN_OPERATOR, None),
      ]
  
      # assemble the root lexing rule. because "|" is ungreedy
      # we have to sort by length so that the lexer continues working
      # as expected when we have parsing rules like <% for block and
      # <%= for variables. (if someone wants asp like syntax)
      # variables are just part of the rules if variable processing
      # is required.
      root_tag_rules = compile_rules(environment)
  
      # block suffix if trimming is enabled
      block_suffix_re = environment.trim_blocks and "\\n?" or ""
  
      # If lstrip is enabled, it should not be applied if there is any
      # non-whitespace between the newline and block.
      self.lstrip_unless_re = (
          c(r"[^ \t]") if environment.lstrip_blocks else None
      )
  
      self.newline_sequence = environment.newline_sequence
      self.keep_trailing_newline = environment.keep_trailing_newline
  
      # global lexing rules
      self.rules = {
          "root": [
              # directives
              (
                  c(
                      "(.*?)(?:%s)"
                      % "|".join(
                          [
                              r"(?P<raw_begin>%s(\-|\+|)\s*raw\s*(?:\-%s\s*|%s))"
                              % (
                                  e(environment.block_start_string),
                                  e(environment.block_end_string),
                                  e(environment.block_end_string),
                              )
                          ]
                          + [
                              r"(?P<%s>%s(\-|\+|))" % (n, r)
                              for n, r in root_tag_rules
                          ]
                      )
                  ),
                  OptionalLStrip(TOKEN_DATA, "#bygroup"),
                  "#bygroup",
              ),
              # data
              (c(".+"), TOKEN_DATA, None),
          ],
          # comments
          TOKEN_COMMENT_BEGIN: [
              (
                  c(
                      r"(.*?)((?:\-%s\s*|%s)%s)"
                      % (
                          e(environment.comment_end_string),
                          e(environment.comment_end_string),
                          block_suffix_re,
                      )
                  ),
                  (TOKEN_COMMENT, TOKEN_COMMENT_END),
                  "#pop",
              ),
              (
                  c("(.)"),
                  (Failure("Missing end of comment tag"),),
                  None,
              ),
          ],
          # blocks
          TOKEN_BLOCK_BEGIN: [
              (
                  c(
                      r"(?:\-%s\s*|%s)%s"
                      % (
                          e(environment.block_end_string),
                          e(environment.block_end_string),
                          block_suffix_re,
                      )
                  ),
                  TOKEN_BLOCK_END,
                  "#pop",
              ),
          ]
          + tag_rules,
          # variables
          TOKEN_VARIABLE_BEGIN: [
              (
                  c(
                      r"\-%s\s*|%s"
                      % (
                          e(environment.variable_end_string),
                          e(environment.variable_end_string),
                      )
                  ),
                  TOKEN_VARIABLE_END,
                  "#pop",
              )
          ]
          + tag_rules,
          # raw block
          TOKEN_RAW_BEGIN: [
              (
                  c(
                      r"(.*?)((?:%s(\-|\+|))\s*endraw\s*(?:\-%s\s*|%s%s))"
                      % (
                          e(environment.block_start_string),
                          e(environment.block_end_string),
                          e(environment.block_end_string),
                          block_suffix_re,
                      )
                  ),
                  OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END),
                  "#pop",
              ),
              (
                  c("(.)"),
                  (Failure("Missing end of raw directive"),),
                  None,
              ),
          ],
          # line statements
          TOKEN_LINESTATEMENT_BEGIN: [
              (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop")
          ]
          + tag_rules,
          # line comments
          TOKEN_LINECOMMENT_BEGIN: [
              (
                  c(r"(.*?)()(?=\n|$)"),
                  (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END),
                  "#pop",
              )
          ],
      }
  
imports: import re
  from ast import literal_eval
  from collections import deque
  from operator import itemgetter
  from ._compat import implements_iterator
  from ._compat import intern
  from ._compat import iteritems
  from ._compat import text_type
  from .exceptions import TemplateSyntaxError
  from .utils import LRUCache
label_desc: Use image node_id 1 to create a new Lexer object from inherited base classes: object with example: obj = Lexer(environment)