class_docstr: Lexer for Typographic Number Theory, as described in the book
  Gödel, Escher, Bach, by Douglas R. Hofstadter
  
  .. versionadded:: 2.7
class_docstr_tok: ["Lexer","for","Typographic","Number","Theory",",","as","described","in","the","book","G\u00f6del",",","Escher",",","Bach",",","by","Douglas","R.","Hofstadter","..","versionadded",":",":","2.7"]
docstr: Returns a list of TNT tokens.
docstr_tok: ["Returns","a","list","of","TNT","tokens","."]
code: def get_tokens_unprocessed(self, text):
      """Returns a list of TNT tokens."""
      self.cur = []
      start = end = self.whitespace(0, text)
      while start <= end < len(text):
          try:
              # try line number
              while text[end] in self.NUMBERS:
                  end += 1
              if end != start:  # actual number present
                  self.cur.append(
                      (start, Number.Integer, text[start:end])
                  )
                  # whitespace is required after a line number
                  orig = len(self.cur)
                  try:
                      start = end = self.whitespace(end, text, True)
                  except AssertionError:
                      del self.cur[orig:]
                      start = end = self.error_till_line_end(end, text)
                      continue
              # at this point it could be a comment
              match = self.COMMENT.match(text, start)
              if match is not None:
                  self.cur.append(
                      (start, Comment, text[start : match.end()])
                  )
                  start = end = match.end()
                  # anything after the closing bracket is invalid
                  start = end = self.error_till_line_end(start, text)
                  # do not attempt to process the rest
                  continue
              del match
              if text[start] in "[]":  # fantasy push or pop
                  self.cur.append((start, Keyword, text[start]))
                  start += 1
                  end += 1
              else:
                  # one formula, possibly containing subformulae
                  orig = len(self.cur)
                  try:
                      start = end = self.formula(start, text)
                  except (
                      AssertionError,
                      RecursionError,
                  ):  # not well-formed
                      del self.cur[orig:]
                      while text[end] not in self.WHITESPACE:
                          end += 1
                      self.cur.append((start, Error, text[start:end]))
                      start = end
              # skip whitespace after formula
              orig = len(self.cur)
              try:
                  start = end = self.whitespace(end, text, True)
              except AssertionError:
                  del self.cur[orig:]
                  start = end = self.error_till_line_end(start, text)
                  continue
              # rule proving this formula a theorem
              orig = len(self.cur)
              try:
                  start = end = self.rule(start, text)
              except AssertionError:
                  del self.cur[orig:]
                  start = end = self.error_till_line_end(start, text)
                  continue
              # skip whitespace after rule
              start = end = self.whitespace(end, text)
              # line marker
              if text[start] == "(":
                  orig = len(self.cur)
                  try:
                      start = end = self.lineno(start, text)
                  except AssertionError:
                      del self.cur[orig:]
                      start = end = self.error_till_line_end(
                          start, text
                      )
                      continue
                  start = end = self.whitespace(start, text)
          except IndexError:
              try:
                  del self.cur[orig:]
              except NameError:
                  pass  # if orig was never defined, fine
              self.error_till_line_end(start, text)
      return self.cur
  
file_path: catboost/contrib/python/Pygments/py3/pygments/lexers/tnt.py
imports: import re
  from pygments.lexer import Lexer
  from pygments.token import Text, Comment, Operator, Keyword, Name, Number, Punctuation, Error
label_desc: Use image node_id 9 for calling the TNTLexer obj's underlying member method code with example usage: obj.get_tokens_unprocessed(text) and returns: self