prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  pytype can't parse hex literals
  The following code causes the error `ParseError: syntax error, unexpected NAME, expecting ']'`:
  
  ```py
  from typing import Literal
  
  a: Literal[0x00ff]
  ```
  Add lexing for base 2, 8, and 16 literals
  Alters lexer.lex and adds a test for the new numeric types, Closes #494 
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `pytype/tools/traces/source.py`
  Content:
  ```
  1 # Lint as: python2, python3
  2 """Source and trace information."""
  3 
  4 from __future__ import print_function
  5 import collections
  6 
  7 Location = collections.namedtuple("Location", ("line", "column"))
  8 
  9 
  10 class AbstractTrace(
  11     collections.namedtuple("AbstractTrace", ("op", "symbol", "types"))):
  12 
  13   def __new__(cls, op, symbol, types):
  14     if cls is AbstractTrace:
  15       raise TypeError("cannot instantiate AbstractTrace")
  16     return super(AbstractTrace, cls).__new__(cls, op, symbol, types)
  17 
  18   def __repr__(self):
  19     return "%s : %s <- %s" % self
  20 
  21 
  22 class Code(object):
  23   """Line-based source code access.
  24 
  25   Attributes:
  26     text: The source text.
  27     traces: A dictionary from line number to traces.
  28     filename: The filename - when using traces.trace(), this value is meaningful
  29       only if an options object containing the filename was provided.
  30   """
  31 
  32   def __init__(self, src, raw_traces, trace_factory, filename):
  33     """Initializer.
  34 
  35     Args:
  36       src: The source text.
  37       raw_traces: Raw (opcode, symbol, types) values.
  38       trace_factory: A subclass of source.AbstractTrace that will be used to
  39         instantiate traces from raw values.
  40       filename: The filename.
  41     """
  42     self.text = src
  43     self.traces = _collect_traces(raw_traces, trace_factory)
  44     self.filename = filename
  45     self._lines = src.split("\n")
  46     self._offsets = []
  47     self._init_byte_offsets()
  48 
  49   def _init_byte_offsets(self):
  50     offset = 0
  51     for line in self._lines:
  52       self._offsets.append(offset)
  53       offset += len(line) + 1  # account for the \n
  54 
  55   def get_offset(self, location):
  56     """Gets the utf-8 byte offset of a source.Location from start of source."""
  57     return self._offsets[location.line - 1] + location.column
  58 
  59   def line(self, n):
  60     """Gets the text at a line number."""
  61     return self._lines[n - 1]
  62 
  63   def get_closest_line_range(self, start, end):
  64     """Gets all valid line numbers in the [start, end) line range."""
  65     return range(start, min(end, len(self._lines) + 1))
  66 
  67   def find_first_text(self, start, end, text):
  68     """Gets first location, if any, the string appears at in the line range."""
  69 
  70     for l in self.get_closest_line_range(start, end):
  71       col = self.line(l).find(text)
  72       if col > -1:
  73         # TODO(mdemello): Temporary hack, replace with a token stream!
  74         # This will break if we have a # in a string before our desired text.
  75         comment_marker = self.line(l).find("#")
  76         if -1 < comment_marker < col:
  77           continue
  78         return Location(l, col)
  79     return None
  80 
  81   def next_non_comment_line(self, line):
  82     """Gets the next non-comment line, if any, after the given line."""
  83     for l in range(line + 1, len(self._lines) + 1):
  84       if self.line(l).lstrip().startswith("#"):
  85         continue
  86       return l
  87     return None
  88 
  89   def display_traces(self):
  90     """Prints the source file with traces for debugging."""
  91     for line in sorted(self.traces):
  92       print("%d %s" % (line, self.line(line)))
  93       for trace in self.traces[line]:
  94         print("  %s" % (trace,))
  95       print("-------------------")
  96 
  97   def get_attr_location(self, name, location):
  98     """Returns the location and span of the attribute in an attribute access.
  99 
  100     Args:
  101       name: The attribute name.
  102       location: The location of the value the attribute is accessed on.
  103     """
  104     # TODO(mdemello): This is pretty crude, and does not for example take into
  105     # account multiple calls of the same attribute in a line. It is just to get
  106     # our tests passing until we incorporate asttokens.
  107     line, _ = location
  108     src_line = self.line(line)
  109     attr = name.split(".")[-1]
  110     dot_attr = "." + attr
  111     if dot_attr in src_line:
  112       col = src_line.index(dot_attr)
  113       return (Location(line, col + 1), len(attr))
  114     else:
  115       # We have something like
  116       #   (foo
  117       #      .bar)
  118       # or
  119       #   (foo.
  120       #     bar)
  121       # Lookahead up to 5 lines to find '.attr' (the ast node always starts from
  122       # the beginning of the chain, so foo.\nbar.\nbaz etc could span several
  123       # lines).
  124       attr_loc = self._get_multiline_location(location, 5, dot_attr)
  125       if attr_loc:
  126         return (Location(attr_loc.line, attr_loc.column + 1), len(attr))
  127       else:
  128         # Find consecutive lines ending with '.' and starting with 'attr'.
  129         for l in self.get_closest_line_range(line, line + 5):
  130           if self.line(l).endswith("."):
  131             next_line = self.next_non_comment_line(l)
  132             text = self.line(next_line)
  133             if text.lstrip().startswith(attr):
  134               c = text.index(attr)
  135               return (Location(next_line, c), len(attr))
  136       # if all else fails, fall back to just spanning the name
  137       return (location, len(name))
  138 
  139   def _get_multiline_location(self, location, n_lines, text):
  140     """Gets the start location of text anywhere within n_lines of location."""
  141     line, _ = location
  142     text_loc = self.find_first_text(line, line + n_lines, text)
  143     if text_loc:
  144       return text_loc
  145     else:
  146       return None
  147 
  148 
  149 def _collect_traces(raw_traces, trace_factory):
  150   """Postprocesses pytype's opcode traces."""
  151   out = collections.defaultdict(list)
  152   for op, symbol, data in raw_traces:
  153     out[op.line].append(trace_factory(op.name, symbol, data))
  154   return out
  155
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/pytype/tools/traces/source.py b/pytype/tools/traces/source.py
  --- a/pytype/tools/traces/source.py
  +++ b/pytype/tools/traces/source.py
  @@ -4,6 +4,9 @@
   from __future__ import print_function
   import collections
   
  +from pytype import compat
  +
  +
   Location = collections.namedtuple("Location", ("line", "column"))
   
   
  @@ -50,7 +53,9 @@
       offset = 0
       for line in self._lines:
         self._offsets.append(offset)
  -      offset += len(line) + 1  # account for the \n
  +      # convert line to bytes
  +      bytes_ = compat.bytestring(line)
  +      offset += len(bytes_) + 1  # account for the \n
   
     def get_offset(self, location):
       """Gets the utf-8 byte offset of a source.Location from start of source."""
  
verification_info: {"golden_diff": "diff --git a/pytype/tools/traces/source.py b/pytype/tools/traces/source.py\n--- a/pytype/tools/traces/source.py\n+++ b/pytype/tools/traces/source.py\n@@ -4,6 +4,9 @@\n from __future__ import print_function\n import collections\n \n+from pytype import compat\n+\n+\n Location = collections.namedtuple(\"Location\", (\"line\", \"column\"))\n \n \n@@ -50,7 +53,9 @@\n     offset = 0\n     for line in self._lines:\n       self._offsets.append(offset)\n-      offset += len(line) + 1  # account for the \\n\n+      # convert line to bytes\n+      bytes_ = compat.bytestring(line)\n+      offset += len(bytes_) + 1  # account for the \\n\n \n   def get_offset(self, location):\n     \"\"\"Gets the utf-8 byte offset of a source.Location from start of source.\"\"\"\n", "issue": "pytype can't parse hex literals\nThe following code causes the error `ParseError: syntax error, unexpected NAME, expecting ']'`:\r\n\r\n```py\r\nfrom typing import Literal\r\n\r\na: Literal[0x00ff]\r\n```\nAdd lexing for base 2, 8, and 16 literals\nAlters lexer.lex and adds a test for the new numeric types, Closes #494 \n", "before_files": [{"content": "# Lint as: python2, python3\n\"\"\"Source and trace information.\"\"\"\n\nfrom __future__ import print_function\nimport collections\n\nLocation = collections.namedtuple(\"Location\", (\"line\", \"column\"))\n\n\nclass AbstractTrace(\n    collections.namedtuple(\"AbstractTrace\", (\"op\", \"symbol\", \"types\"))):\n\n  def __new__(cls, op, symbol, types):\n    if cls is AbstractTrace:\n      raise TypeError(\"cannot instantiate AbstractTrace\")\n    return super(AbstractTrace, cls).__new__(cls, op, symbol, types)\n\n  def __repr__(self):\n    return \"%s : %s <- %s\" % self\n\n\nclass Code(object):\n  \"\"\"Line-based source code access.\n\n  Attributes:\n    text: The source text.\n    traces: A dictionary from line number to traces.\n    filename: The filename - when using traces.trace(), this value is meaningful\n      only if an options object containing the filename was provided.\n  \"\"\"\n\n  def __init__(self, src, raw_traces, trace_factory, filename):\n    \"\"\"Initializer.\n\n    Args:\n      src: The source text.\n      raw_traces: Raw (opcode, symbol, types) values.\n      trace_factory: A subclass of source.AbstractTrace that will be used to\n        instantiate traces from raw values.\n      filename: The filename.\n    \"\"\"\n    self.text = src\n    self.traces = _collect_traces(raw_traces, trace_factory)\n    self.filename = filename\n    self._lines = src.split(\"\\n\")\n    self._offsets = []\n    self._init_byte_offsets()\n\n  def _init_byte_offsets(self):\n    offset = 0\n    for line in self._lines:\n      self._offsets.append(offset)\n      offset += len(line) + 1  # account for the \\n\n\n  def get_offset(self, location):\n    \"\"\"Gets the utf-8 byte offset of a source.Location from start of source.\"\"\"\n    return self._offsets[location.line - 1] + location.column\n\n  def line(self, n):\n    \"\"\"Gets the text at a line number.\"\"\"\n    return self._lines[n - 1]\n\n  def get_closest_line_range(self, start, end):\n    \"\"\"Gets all valid line numbers in the [start, end) line range.\"\"\"\n    return range(start, min(end, len(self._lines) + 1))\n\n  def find_first_text(self, start, end, text):\n    \"\"\"Gets first location, if any, the string appears at in the line range.\"\"\"\n\n    for l in self.get_closest_line_range(start, end):\n      col = self.line(l).find(text)\n      if col > -1:\n        # TODO(mdemello): Temporary hack, replace with a token stream!\n        # This will break if we have a # in a string before our desired text.\n        comment_marker = self.line(l).find(\"#\")\n        if -1 < comment_marker < col:\n          continue\n        return Location(l, col)\n    return None\n\n  def next_non_comment_line(self, line):\n    \"\"\"Gets the next non-comment line, if any, after the given line.\"\"\"\n    for l in range(line + 1, len(self._lines) + 1):\n      if self.line(l).lstrip().startswith(\"#\"):\n        continue\n      return l\n    return None\n\n  def display_traces(self):\n    \"\"\"Prints the source file with traces for debugging.\"\"\"\n    for line in sorted(self.traces):\n      print(\"%d %s\" % (line, self.line(line)))\n      for trace in self.traces[line]:\n        print(\"  %s\" % (trace,))\n      print(\"-------------------\")\n\n  def get_attr_location(self, name, location):\n    \"\"\"Returns the location and span of the attribute in an attribute access.\n\n    Args:\n      name: The attribute name.\n      location: The location of the value the attribute is accessed on.\n    \"\"\"\n    # TODO(mdemello): This is pretty crude, and does not for example take into\n    # account multiple calls of the same attribute in a line. It is just to get\n    # our tests passing until we incorporate asttokens.\n    line, _ = location\n    src_line = self.line(line)\n    attr = name.split(\".\")[-1]\n    dot_attr = \".\" + attr\n    if dot_attr in src_line:\n      col = src_line.index(dot_attr)\n      return (Location(line, col + 1), len(attr))\n    else:\n      # We have something like\n      #   (foo\n      #      .bar)\n      # or\n      #   (foo.\n      #     bar)\n      # Lookahead up to 5 lines to find '.attr' (the ast node always starts from\n      # the beginning of the chain, so foo.\\nbar.\\nbaz etc could span several\n      # lines).\n      attr_loc = self._get_multiline_location(location, 5, dot_attr)\n      if attr_loc:\n        return (Location(attr_loc.line, attr_loc.column + 1), len(attr))\n      else:\n        # Find consecutive lines ending with '.' and starting with 'attr'.\n        for l in self.get_closest_line_range(line, line + 5):\n          if self.line(l).endswith(\".\"):\n            next_line = self.next_non_comment_line(l)\n            text = self.line(next_line)\n            if text.lstrip().startswith(attr):\n              c = text.index(attr)\n              return (Location(next_line, c), len(attr))\n      # if all else fails, fall back to just spanning the name\n      return (location, len(name))\n\n  def _get_multiline_location(self, location, n_lines, text):\n    \"\"\"Gets the start location of text anywhere within n_lines of location.\"\"\"\n    line, _ = location\n    text_loc = self.find_first_text(line, line + n_lines, text)\n    if text_loc:\n      return text_loc\n    else:\n      return None\n\n\ndef _collect_traces(raw_traces, trace_factory):\n  \"\"\"Postprocesses pytype's opcode traces.\"\"\"\n  out = collections.defaultdict(list)\n  for op, symbol, data in raw_traces:\n    out[op.line].append(trace_factory(op.name, symbol, data))\n  return out\n", "path": "pytype/tools/traces/source.py"}], "after_files": [{"content": "# Lint as: python2, python3\n\"\"\"Source and trace information.\"\"\"\n\nfrom __future__ import print_function\nimport collections\n\nfrom pytype import compat\n\n\nLocation = collections.namedtuple(\"Location\", (\"line\", \"column\"))\n\n\nclass AbstractTrace(\n    collections.namedtuple(\"AbstractTrace\", (\"op\", \"symbol\", \"types\"))):\n\n  def __new__(cls, op, symbol, types):\n    if cls is AbstractTrace:\n      raise TypeError(\"cannot instantiate AbstractTrace\")\n    return super(AbstractTrace, cls).__new__(cls, op, symbol, types)\n\n  def __repr__(self):\n    return \"%s : %s <- %s\" % self\n\n\nclass Code(object):\n  \"\"\"Line-based source code access.\n\n  Attributes:\n    text: The source text.\n    traces: A dictionary from line number to traces.\n    filename: The filename - when using traces.trace(), this value is meaningful\n      only if an options object containing the filename was provided.\n  \"\"\"\n\n  def __init__(self, src, raw_traces, trace_factory, filename):\n    \"\"\"Initializer.\n\n    Args:\n      src: The source text.\n      raw_traces: Raw (opcode, symbol, types) values.\n      trace_factory: A subclass of source.AbstractTrace that will be used to\n        instantiate traces from raw values.\n      filename: The filename.\n    \"\"\"\n    self.text = src\n    self.traces = _collect_traces(raw_traces, trace_factory)\n    self.filename = filename\n    self._lines = src.split(\"\\n\")\n    self._offsets = []\n    self._init_byte_offsets()\n\n  def _init_byte_offsets(self):\n    offset = 0\n    for line in self._lines:\n      self._offsets.append(offset)\n      # convert line to bytes\n      bytes_ = compat.bytestring(line)\n      offset += len(bytes_) + 1  # account for the \\n\n\n  def get_offset(self, location):\n    \"\"\"Gets the utf-8 byte offset of a source.Location from start of source.\"\"\"\n    return self._offsets[location.line - 1] + location.column\n\n  def line(self, n):\n    \"\"\"Gets the text at a line number.\"\"\"\n    return self._lines[n - 1]\n\n  def get_closest_line_range(self, start, end):\n    \"\"\"Gets all valid line numbers in the [start, end) line range.\"\"\"\n    return range(start, min(end, len(self._lines) + 1))\n\n  def find_first_text(self, start, end, text):\n    \"\"\"Gets first location, if any, the string appears at in the line range.\"\"\"\n\n    for l in self.get_closest_line_range(start, end):\n      col = self.line(l).find(text)\n      if col > -1:\n        # TODO(mdemello): Temporary hack, replace with a token stream!\n        # This will break if we have a # in a string before our desired text.\n        comment_marker = self.line(l).find(\"#\")\n        if -1 < comment_marker < col:\n          continue\n        return Location(l, col)\n    return None\n\n  def next_non_comment_line(self, line):\n    \"\"\"Gets the next non-comment line, if any, after the given line.\"\"\"\n    for l in range(line + 1, len(self._lines) + 1):\n      if self.line(l).lstrip().startswith(\"#\"):\n        continue\n      return l\n    return None\n\n  def display_traces(self):\n    \"\"\"Prints the source file with traces for debugging.\"\"\"\n    for line in sorted(self.traces):\n      print(\"%d %s\" % (line, self.line(line)))\n      for trace in self.traces[line]:\n        print(\"  %s\" % (trace,))\n      print(\"-------------------\")\n\n  def get_attr_location(self, name, location):\n    \"\"\"Returns the location and span of the attribute in an attribute access.\n\n    Args:\n      name: The attribute name.\n      location: The location of the value the attribute is accessed on.\n    \"\"\"\n    # TODO(mdemello): This is pretty crude, and does not for example take into\n    # account multiple calls of the same attribute in a line. It is just to get\n    # our tests passing until we incorporate asttokens.\n    line, _ = location\n    src_line = self.line(line)\n    attr = name.split(\".\")[-1]\n    dot_attr = \".\" + attr\n    if dot_attr in src_line:\n      col = src_line.index(dot_attr)\n      return (Location(line, col + 1), len(attr))\n    else:\n      # We have something like\n      #   (foo\n      #      .bar)\n      # or\n      #   (foo.\n      #     bar)\n      # Lookahead up to 5 lines to find '.attr' (the ast node always starts from\n      # the beginning of the chain, so foo.\\nbar.\\nbaz etc could span several\n      # lines).\n      attr_loc = self._get_multiline_location(location, 5, dot_attr)\n      if attr_loc:\n        return (Location(attr_loc.line, attr_loc.column + 1), len(attr))\n      else:\n        # Find consecutive lines ending with '.' and starting with 'attr'.\n        for l in self.get_closest_line_range(line, line + 5):\n          if self.line(l).endswith(\".\"):\n            next_line = self.next_non_comment_line(l)\n            text = self.line(next_line)\n            if text.lstrip().startswith(attr):\n              c = text.index(attr)\n              return (Location(next_line, c), len(attr))\n      # if all else fails, fall back to just spanning the name\n      return (location, len(name))\n\n  def _get_multiline_location(self, location, n_lines, text):\n    \"\"\"Gets the start location of text anywhere within n_lines of location.\"\"\"\n    line, _ = location\n    text_loc = self.find_first_text(line, line + n_lines, text)\n    if text_loc:\n      return text_loc\n    else:\n      return None\n\n\ndef _collect_traces(raw_traces, trace_factory):\n  \"\"\"Postprocesses pytype's opcode traces.\"\"\"\n  out = collections.defaultdict(list)\n  for op, symbol, data in raw_traces:\n    out[op.line].append(trace_factory(op.name, symbol, data))\n  return out\n", "path": "pytype/tools/traces/source.py"}]}