class_docstr: Class that implements a lexer for a given environment. Automatically
  created by the environment class, usually you don't have to do that.
  
  Note that the lexer is not automatically bound to an environment.
  Multiple environments can share the same lexer.
class_docstr_tok: ["Class","that","implements","a","lexer","for","a","given","environment",".","Automatically","created","by","the","environment","class",",","usually","you","do","n't","have","to","do","that",".","Note","that","the","lexer","is","not","automatically","bound","to","an","environment",".","Multiple","environments","can","share","the","same","lexer","."]
docstr: This method tokenizes the text and returns the tokens in a
  generator.  Use this method if you just want to tokenize a template.
docstr_tok: ["This","method","tokenizes","the","text","and","returns","the","tokens","in","a","generator",".","Use","this","method","if","you","just","want","to","tokenize","a","template","."]
code: def tokeniter(self, source, name, filename=None, state=None):
      """This method tokenizes the text and returns the tokens in a
      generator.  Use this method if you just want to tokenize a template.
      """
      source = text_type(source)
      lines = source.splitlines()
      if self.keep_trailing_newline and source:
          for newline in ("\r\n", "\r", "\n"):
              if source.endswith(newline):
                  lines.append("")
                  break
      source = "\n".join(lines)
      pos = 0
      lineno = 1
      stack = ["root"]
      if state is not None and state != "root":
          assert state in ("variable", "block"), "invalid state"
          stack.append(state + "_begin")
      statetokens = self.rules[stack[-1]]
      source_length = len(source)
      balancing_stack = []
      lstrip_unless_re = self.lstrip_unless_re
      newlines_stripped = 0
      line_starting = True
  
      while 1:
          # tokenizer loop
          for regex, tokens, new_state in statetokens:
              m = regex.match(source, pos)
              # if no match we try again with the next rule
              if m is None:
                  continue
  
              # we only match blocks and variables if braces / parentheses
              # are balanced. continue parsing with the lower rule which
              # is the operator rule. do this only if the end tags look
              # like operators
              if balancing_stack and tokens in (
                  TOKEN_VARIABLE_END,
                  TOKEN_BLOCK_END,
                  TOKEN_LINESTATEMENT_END,
              ):
                  continue
  
              # tuples support more options
              if isinstance(tokens, tuple):
                  groups = m.groups()
  
                  if isinstance(tokens, OptionalLStrip):
                      # Rule supports lstrip. Match will look like
                      # text, block type, whitespace control, type, control, ...
                      text = groups[0]
  
                      # Skipping the text and first type, every other group is the
                      # whitespace control for each type. One of the groups will be
                      # -, +, or empty string instead of None.
                      strip_sign = next(
                          g for g in groups[2::2] if g is not None
                      )
  
                      if strip_sign == "-":
                          # Strip all whitespace between the text and the tag.
                          stripped = text.rstrip()
                          newlines_stripped = text[
                              len(stripped) :
                          ].count("\n")
                          groups = (stripped,) + groups[1:]
                      elif (
                          # Not marked for preserving whitespace.
                          strip_sign != "+"
                          # lstrip is enabled.
                          and lstrip_unless_re is not None
                          # Not a variable expression.
                          and not m.groupdict().get(
                              TOKEN_VARIABLE_BEGIN
                          )
                      ):
                          # The start of text between the last newline and the tag.
                          l_pos = text.rfind("\n") + 1
                          if l_pos > 0 or line_starting:
                              # If there's only whitespace between the newline and the
                              # tag, strip it.
                              if not lstrip_unless_re.search(
                                  text, l_pos
                              ):
                                  groups = (text[:l_pos],) + groups[1:]
  
                  for idx, token in enumerate(tokens):
                      # failure group
                      if token.__class__ is Failure:
                          raise token(lineno, filename)
                      # bygroup is a bit more complex, in that case we
                      # yield for the current token the first named
                      # group that matched
                      elif token == "#bygroup":
                          for key, value in iteritems(m.groupdict()):
                              if value is not None:
                                  yield lineno, key, value
                                  lineno += value.count("\n")
                                  break
                          else:
                              raise RuntimeError(
                                  "%r wanted to resolve "
                                  "the token dynamically"
                                  " but no group matched" % regex
                              )
                      # normal group
                      else:
                          data = groups[idx]
                          if data or token not in ignore_if_empty:
                              yield lineno, token, data
                          lineno += data.count("\n") + newlines_stripped
                          newlines_stripped = 0
  
              # strings as token just are yielded as it.
              else:
                  data = m.group()
                  # update brace/parentheses balance
                  if tokens == TOKEN_OPERATOR:
                      if data == "{":
                          balancing_stack.append("}")
                      elif data == "(":
                          balancing_stack.append(")")
                      elif data == "[":
                          balancing_stack.append("]")
                      elif data in ("}", ")", "]"):
                          if not balancing_stack:
                              raise TemplateSyntaxError(
                                  "unexpected '%s'" % data,
                                  lineno,
                                  name,
                                  filename,
                              )
                          expected_op = balancing_stack.pop()
                          if expected_op != data:
                              raise TemplateSyntaxError(
                                  "unexpected '%s', "
                                  "expected '%s'" % (data, expected_op),
                                  lineno,
                                  name,
                                  filename,
                              )
                  # yield items
                  if data or tokens not in ignore_if_empty:
                      yield lineno, tokens, data
                  lineno += data.count("\n")
  
              line_starting = m.group()[-1:] == "\n"
  
              # fetch new position into new variable so that we can check
              # if there is a internal parsing error which would result
              # in an infinite loop
              pos2 = m.end()
  
              # handle state changes
              if new_state is not None:
                  # remove the uppermost state
                  if new_state == "#pop":
                      stack.pop()
                  # resolve the new state by group checking
                  elif new_state == "#bygroup":
                      for key, value in iteritems(m.groupdict()):
                          if value is not None:
                              stack.append(key)
                              break
                      else:
                          raise RuntimeError(
                              "%r wanted to resolve the "
                              "new state dynamically but"
                              " no group matched" % regex
                          )
                  # direct state name given
                  else:
                      stack.append(new_state)
                  statetokens = self.rules[stack[-1]]
              # we are still at the same position and no stack change.
              # this means a loop without break condition, avoid that and
              # raise error
              elif pos2 == pos:
                  raise RuntimeError(
                      "%r yielded empty string without stack change"
                      % regex
                  )
              # publish new function and start again
              pos = pos2
              break
          # if loop terminated without break we haven't found a single match
          # either we are at the end of the file or we have a problem
          else:
              # end of text
              if pos >= source_length:
                  return
              # something went wrong
              raise TemplateSyntaxError(
                  "unexpected char %r at %d" % (source[pos], pos),
                  lineno,
                  name,
                  filename,
              )
  
imports: import re
  from ast import literal_eval
  from collections import deque
  from operator import itemgetter
  from ._compat import implements_iterator
  from ._compat import intern
  from ._compat import iteritems
  from ._compat import text_type
  from .exceptions import TemplateSyntaxError
  from .utils import LRUCache
label_desc: Use image node_id 5 for calling the Lexer obj's underlying member method code with example usage: obj.tokeniter(source, name, filename, state) without return types