class_docstr: For Delphi (Borland Object Pascal),
  Turbo Pascal and Free Pascal source code.
  
  Additional options accepted:
  
  `turbopascal`
      Highlight Turbo Pascal specific keywords (default: ``True``).
  `delphi`
      Highlight Borland Delphi specific keywords (default: ``True``).
  `freepascal`
      Highlight Free Pascal specific keywords (default: ``True``).
  `units`
      A list of units that should be considered builtin, supported are
      ``System``, ``SysUtils``, ``Classes`` and ``Math``.
      Default is to consider all of them builtin.
class_docstr_tok: ["For","Delphi","(","Borland","Object","Pascal",")",",","Turbo","Pascal","and","Free","Pascal","source","code",".","Additional","options","accepted",":","`","turbopascal","`","Highlight","Turbo","Pascal","specific","keywords","(","default",":","``","True","``",")",".","`","delphi","`","Highlight","Borland","Delphi","specific","keywords","(","default",":","``","True","``",")",".","`","freepascal","`","Highlight","Free","Pascal","specific","keywords","(","default",":","``","True","``",")",".","`","units","`","A","list","of","units","that","should","be","considered","builtin",",","supported","are","``","System","``",",","``","SysUtils","``",",","``","Classes","``","and","``","Math","``",".","Default","is","to","consider","all","of","them","builtin","."]
code: def get_tokens_unprocessed(self, text):
      scanner = Scanner(text, re.DOTALL | re.MULTILINE | re.IGNORECASE)
      stack = ["initial"]
      in_function_block = False
      in_property_block = False
      was_dot = False
      next_token_is_function = False
      next_token_is_property = False
      collect_labels = False
      block_labels = set()
      brace_balance = [0, 0]
  
      while not scanner.eos:
          token = Error
  
          if stack[-1] == "initial":
              if scanner.scan(r"\s+"):
                  token = Whitespace
              elif not self.is_portugol and scanner.scan(
                  r"\{.*?\}|\(\*.*?\*\)"
              ):
                  if scanner.match.startswith("$"):
                      token = Comment.Preproc
                  else:
                      token = Comment.Multiline
              elif scanner.scan(r"//.*?$"):
                  token = Comment.Single
              elif self.is_portugol and scanner.scan(
                  r"(<\-)|(>=)|(<=)|%|<|>|-|\+|\*|\=|(<>)|\/|\.|:|,"
              ):
                  token = Operator
              elif not self.is_portugol and scanner.scan(
                  r"[-+*\/=<>:;,.@\^]"
              ):
                  token = Operator
                  # stop label highlighting on next ";"
                  if collect_labels and scanner.match == ";":
                      collect_labels = False
              elif scanner.scan(r"[\(\)\[\]]+"):
                  token = Punctuation
                  # abort function naming ``foo = Function(...)``
                  next_token_is_function = False
                  # if we are in a function block we count the open
                  # braces because ootherwise it's impossible to
                  # determine the end of the modifier context
                  if in_function_block or in_property_block:
                      if scanner.match == "(":
                          brace_balance[0] += 1
                      elif scanner.match == ")":
                          brace_balance[0] -= 1
                      elif scanner.match == "[":
                          brace_balance[1] += 1
                      elif scanner.match == "]":
                          brace_balance[1] -= 1
              elif scanner.scan(r"[A-Za-z_][A-Za-z_0-9]*"):
                  lowercase_name = scanner.match.lower()
                  if lowercase_name == "result":
                      token = Name.Builtin.Pseudo
                  elif lowercase_name in self.keywords:
                      token = Keyword
                      # if we are in a special block and a
                      # block ending keyword occurs (and the parenthesis
                      # is balanced) we end the current block context
                      if self.is_portugol:
                          if lowercase_name in (
                              "funcao",
                              "procedimento",
                          ):
                              in_function_block = True
                              next_token_is_function = True
                      else:
                          if (
                              (in_function_block or in_property_block)
                              and lowercase_name in self.BLOCK_KEYWORDS
                              and brace_balance[0] <= 0
                              and brace_balance[1] <= 0
                          ):
                              in_function_block = False
                              in_property_block = False
                              brace_balance = [0, 0]
                              block_labels = set()
                          if lowercase_name in ("label", "goto"):
                              collect_labels = True
                          elif lowercase_name == "asm":
                              stack.append("asm")
                          elif lowercase_name == "property":
                              in_property_block = True
                              next_token_is_property = True
                          elif lowercase_name in (
                              "procedure",
                              "operator",
                              "function",
                              "constructor",
                              "destructor",
                          ):
                              in_function_block = True
                              next_token_is_function = True
                  # we are in a function block and the current name
                  # is in the set of registered modifiers. highlight
                  # it as pseudo keyword
                  elif (
                      not self.is_portugol
                      and in_function_block
                      and lowercase_name in self.FUNCTION_MODIFIERS
                  ):
                      token = Keyword.Pseudo
                  # if we are in a property highlight some more
                  # modifiers
                  elif (
                      not self.is_portugol
                      and in_property_block
                      and lowercase_name in ("read", "write")
                  ):
                      token = Keyword.Pseudo
                      next_token_is_function = True
                  # if the last iteration set next_token_is_function
                  # to true we now want this name highlighted as
                  # function. so do that and reset the state
                  elif next_token_is_function:
                      # Look if the next token is a dot. If yes it's
                      # not a function, but a class name and the
                      # part after the dot a function name
                      if not self.is_portugol and scanner.test(
                          r"\s*\.\s*"
                      ):
                          token = Name.Class
                      # it's not a dot, our job is done
                      else:
                          token = Name.Function
                          next_token_is_function = False
  
                          if self.is_portugol:
                              block_labels.add(scanner.match.lower())
  
                  # same for properties
                  elif not self.is_portugol and next_token_is_property:
                      token = Name.Property
                      next_token_is_property = False
                  # Highlight this token as label and add it
                  # to the list of known labels
                  elif not self.is_portugol and collect_labels:
                      token = Name.Label
                      block_labels.add(scanner.match.lower())
                  # name is in list of known labels
                  elif lowercase_name in block_labels:
                      token = Name.Label
                  elif (
                      self.is_portugol
                      and lowercase_name in self.PORTUGOL_BUILTIN_TYPES
                  ):
                      token = Keyword.Type
                  elif (
                      not self.is_portugol
                      and lowercase_name in self.BUILTIN_TYPES
                  ):
                      token = Keyword.Type
                  elif (
                      not self.is_portugol
                      and lowercase_name in self.DIRECTIVES
                  ):
                      token = Keyword.Pseudo
                  # builtins are just builtins if the token
                  # before isn't a dot
                  elif (
                      not self.is_portugol
                      and not was_dot
                      and lowercase_name in self.builtins
                  ):
                      token = Name.Builtin
                  else:
                      token = Name
              elif self.is_portugol and scanner.scan(r"\""):
                  token = String
                  stack.append("string")
              elif not self.is_portugol and scanner.scan(r"'"):
                  token = String
                  stack.append("string")
              elif not self.is_portugol and scanner.scan(
                  r"\#(\d+|\$[0-9A-Fa-f]+)"
              ):
                  token = String.Char
              elif not self.is_portugol and scanner.scan(
                  r"\$[0-9A-Fa-f]+"
              ):
                  token = Number.Hex
              elif scanner.scan(r"\d+(?![eE]|\.[^.])"):
                  token = Number.Integer
              elif scanner.scan(
                  r"\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)"
              ):
                  token = Number.Float
              else:
                  # if the stack depth is deeper than once, pop
                  if len(stack) > 1:
                      stack.pop()
                  scanner.get_char()
  
          elif stack[-1] == "string":
              if self.is_portugol:
                  if scanner.scan(r"''"):
                      token = String.Escape
                  elif scanner.scan(r"\""):
                      token = String
                      stack.pop()
                  elif scanner.scan(r"[^\"]*"):
                      token = String
                  else:
                      scanner.get_char()
                      stack.pop()
              else:
                  if scanner.scan(r"''"):
                      token = String.Escape
                  elif scanner.scan(r"'"):
                      token = String
                      stack.pop()
                  elif scanner.scan(r"[^']*"):
                      token = String
                  else:
                      scanner.get_char()
                      stack.pop()
          elif not self.is_portugol and stack[-1] == "asm":
              if scanner.scan(r"\s+"):
                  token = Whitespace
              elif scanner.scan(r"end"):
                  token = Keyword
                  stack.pop()
              elif scanner.scan(r"\{.*?\}|\(\*.*?\*\)"):
                  if scanner.match.startswith("$"):
                      token = Comment.Preproc
                  else:
                      token = Comment.Multiline
              elif scanner.scan(r"//.*?$"):
                  token = Comment.Single
              elif scanner.scan(r"'"):
                  token = String
                  stack.append("string")
              elif scanner.scan(r"@@[A-Za-z_][A-Za-z_0-9]*"):
                  token = Name.Label
              elif scanner.scan(r"[A-Za-z_][A-Za-z_0-9]*"):
                  lowercase_name = scanner.match.lower()
                  if lowercase_name in self.ASM_INSTRUCTIONS:
                      token = Keyword
                  elif lowercase_name in self.ASM_REGISTERS:
                      token = Name.Builtin
                  else:
                      token = Name
              elif scanner.scan(r"[-+*\/=<>:;,.@\^]+"):
                  token = Operator
              elif scanner.scan(r"[\(\)\[\]]+"):
                  token = Punctuation
              elif scanner.scan(r"\$[0-9A-Fa-f]+"):
                  token = Number.Hex
              elif scanner.scan(r"\d+(?![eE]|\.[^.])"):
                  token = Number.Integer
              elif scanner.scan(
                  r"\d+(\.\d+([eE][+-]?\d+)?|[eE][+-]?\d+)"
              ):
                  token = Number.Float
              else:
                  scanner.get_char()
                  stack.pop()
  
          # save the dot!!!11
          if not self.is_portugol and scanner.match.strip():
              was_dot = scanner.match == "."
  
          yield scanner.start_pos, token, scanner.match or ""
  
file_path: catboost/contrib/python/Pygments/py3/pygments/lexers/pascal.py
imports: import re
  from pygments.lexer import Lexer
  from pygments.util import get_bool_opt, get_list_opt
  from pygments.token import Text, Comment, Operator, Keyword, Name, String, Number, Punctuation, Error, Whitespace
  from pygments.scanner import Scanner
  from pygments.lexers.modula2 import Modula2Lexer
label_desc: Use image node_id 2 for calling the DelphiLexer obj's underlying member method code with example usage: obj.get_tokens_unprocessed(text) without return types