text: #!/usr/bin/env python3
  # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4
  from __future__ import print_function
  
  import urllib.parse
  import glob
  import os
  import sys
  
  verbose = '--verbose' in sys.argv
  dump = '--dump' in sys.argv
  internal = '--internal' in sys.argv
  plain_output = '--plain-output' in sys.argv
  single_page_output = '--single-page' in sys.argv
  if plain_output:
      plain_file = open('plain_text_out.txt', 'w+')
  in_code = None
  
  paths = ['include/libtorrent/*.hpp', 'include/libtorrent/kademlia/*.hpp', 'include/libtorrent/extensions/*.hpp']
  
  if internal:
      paths.append('include/libtorrent/aux_/*.hpp')
  
  files = []
  
  for p in paths:
      files.extend(glob.glob(os.path.join('..', p)))
  
  functions = []
  classes = []
  enums = []
  constants = {}
  
  # maps filename to overview description
  overviews = {}
  
  # maps names -> URL
  symbols = {}
  
  # some files that need pre-processing to turn symbols into
  # links into the reference documentation
  preprocess_rst = \
      {
          'manual.rst': 'manual-ref.rst',
          'tuning.rst': 'tuning-ref.rst',
          'tutorial.rst': 'tutorial-ref.rst',
          'features.rst': 'features-ref.rst',
          'upgrade_to_1.2.rst': 'upgrade_to_1.2-ref.rst',
          'upgrade_to_2.0.rst': 'upgrade_to_2.0-ref.rst',
          'settings.rst': 'settings-ref.rst'
      }
  
  # some pre-defined sections from the main manual
  symbols = \
      {
          "queuing_": "manual-ref.html#queuing",
          "fast-resume_": "manual-ref.html#fast-resume",
          "storage-allocation_": "manual-ref.html#storage-allocation",
          "alerts_": "manual-ref.html#alerts",
          "upnp-and-nat-pmp_": "manual-ref.html#upnp-and-nat-pmp",
          "http-seeding_": "manual-ref.html#http-seeding",
          "metadata-from-peers_": "manual-ref.html#metadata-from-peers",
          "magnet-links_": "manual-ref.html#magnet-links",
          "ssl-torrents_": "manual-ref.html#ssl-torrents",
          "dynamic-loading-of-torrent-files_": "manual-ref.html#dynamic-loading-of-torrent-files",
          "session-statistics_": "manual-ref.html#session-statistics",
          "peer-classes_": "manual-ref.html#peer-classes"
      }
  
  # parse out names of settings, and add them to the symbols list, to get cross
  # references working
  with open('../src/settings_pack.cpp') as f:
      for line in f:
          line = line.strip()
          if not line.startswith('SET('):
              continue
  
          name = line.split('(')[1].split(',')[0]
          symbols['settings_pack::' + name] = 'reference-Settings.html#' + name
  
  static_links = \
      {
          ".. _`BEP 3`: https://www.bittorrent.org/beps/bep_0003.html",
          ".. _`BEP 17`: https://www.bittorrent.org/beps/bep_0017.html",
          ".. _`BEP 19`: https://www.bittorrent.org/beps/bep_0019.html",
          ".. _`BEP 38`: https://www.bittorrent.org/beps/bep_0038.html",
          ".. _`BEP 42`: https://www.bittorrent.org/beps/bep_0042.html",
          ".. _`rate based choking`: manual-ref.html#rate-based-choking",
          ".. _extensions: manual-ref.html#extensions",
      }
  
  anon_index = 0
  
  category_mapping = {
      'ed25519.hpp': 'ed25519',
      'session.hpp': 'Session',
      'session_handle.hpp': 'Session',
      'torrent_handle.hpp': 'Torrent Handle',
      'torrent_info.hpp': 'Torrent Info',
      'announce_entry.hpp': 'Trackers',
      'peer_class_type_filter.hpp': 'PeerClass',
      'peer_class.hpp': 'PeerClass',
      'torrent_status.hpp': 'Torrent Status',
      'session_stats.hpp': 'Stats',
      'performance_counters.hpp': 'Stats',
      'read_resume_data.hpp': 'Resume Data',
      'write_resume_data.hpp': 'Resume Data',
      'add_torrent_params.hpp': 'Add Torrent',
      'client_data.hpp': 'Add Torrent',
      'session_status.hpp': 'Session',
      'session_params.hpp': 'Session',
      'error_code.hpp': 'Error Codes',
      'storage_defs.hpp': 'Storage',
      'file_storage.hpp': 'Storage',
      'disk_interface.hpp': 'Custom Storage',
      'disk_observer.hpp': 'Custom Storage',
      'mmap_disk_io.hpp': 'Storage',
      'disabled_disk_io.hpp': 'Storage',
      'posix_disk_io.hpp': 'Storage',
      'extensions.hpp': 'Plugins',
      'ut_metadata.hpp': 'Plugins',
      'ut_pex.hpp': 'Plugins',
      'ut_trackers.hpp': 'Plugins',
      'smart_ban.hpp': 'Plugins',
      'peer_connection_handle.hpp': 'Plugins',
      'create_torrent.hpp': 'Create Torrents',
      'alert.hpp': 'Alerts',
      'alert_types.hpp': 'Alerts',
      'bencode.hpp': 'Bencoding',
      'bdecode.hpp': 'Bdecoding',
      'entry.hpp': 'Bencoding',
      'time.hpp': 'Time',
      'escape_string.hpp': 'Utility',
      'enum_net.hpp': 'Network',
      'socket.hpp': 'Network',
      'address.hpp': 'Network',
      'socket_io.hpp': 'Network',
      'bitfield.hpp': 'Utility',
      'sha1_hash.hpp': 'Utility',
      'hasher.hpp': 'Utility',
      'identify_client.hpp': 'Utility',
      'ip_filter.hpp': 'Filter',
      'session_settings.hpp': 'Settings',
      'settings_pack.hpp': 'Settings',
      'fingerprint.hpp': 'Settings',
      'operations.hpp': 'Alerts',
      'disk_buffer_holder.hpp': 'Custom Storage',
      'alert_dispatcher.hpp': 'Alerts',
  }
  
  category_fun_mapping = {
      'min_memory_usage()': 'Settings',
      'high_performance_seed()': 'Settings',
      'default_disk_io_constructor()': 'Storage',
      'settings_interface': 'Custom Storage',
  }
  
  
  def categorize_symbol(name, filename):
      f = os.path.split(filename)[1]
  
      if name.endswith('_category()') \
              or name.endswith('_error_code') \
              or name.endswith('error_code_enum') \
              or name.endswith('errors'):
          return 'Error Codes'
  
      if name in category_fun_mapping:
          return category_fun_mapping[name]
  
      if f in category_mapping:
          return category_mapping[f]
  
      if filename.startswith('libtorrent/kademlia/'):
          return 'DHT'
  
      return 'Core'
  
  
  def suppress_warning(filename, name):
      f = os.path.split(filename)[1]
      if f != 'alert_types.hpp':
          return False
  
      # if name.endswith('_alert') or name == 'message()':
      return True
  
      # return False
  
  
  def first_item(itr):
      for i in itr:
          return i
      return None
  
  
  def is_visible(desc):
      if desc.strip().startswith('hidden'):
          return False
      if internal:
          return True
      if desc.strip().startswith('internal'):
          return False
      return True
  
  
  def highlight_signature(s):
      name = s.split('(', 1)
      name2 = name[0].split(' ')
      if len(name2[-1]) == 0:
          return s
  
      # make the name of the function bold
      name2[-1] = '**' + name2[-1] + '** '
  
      # if there is a return value, make sure we preserve pointer types
      if len(name2) > 1:
          name2[0] = name2[0].replace('*', '\\*')
      name[0] = ' '.join(name2)
  
      # we have to escape asterisks, since this is rendered into
      # a parsed literal in rst
      name[1] = name[1].replace('*', '\\*')
  
      # we also have to escape colons
      name[1] = name[1].replace(':', '\\:')
  
      # escape trailing underscores
      name[1] = name[1].replace('_', '\\_')
  
      # comments in signatures are italic
      name[1] = name[1].replace('/\\*', '*/\\*')
      name[1] = name[1].replace('\\*/', '\\*/*')
      return '('.join(name)
  
  
  def highlight_name(s):
      if '=' in s:
          splitter = ' = '
      elif '{' in s:
          splitter = '{'
      else:
          return s
  
      name = s.split(splitter, 1)
      name2 = name[0].split(' ')
      if len(name2[-1]) == 0:
          return s
  
      name2[-1] = '**' + name2[-1] + '** '
      name[0] = ' '.join(name2)
      return splitter.join(name)
  
  
  def html_sanitize(s):
      ret = ''
      for i in s:
          if i == '<':
              ret += '&lt;'
          elif i == '>':
              ret += '&gt;'
          elif i == '&':
              ret += '&amp;'
          else:
              ret += i
      return ret
  
  
  def looks_like_namespace(line):
      line = line.strip()
      if line.startswith('namespace'):
          return True
      return False
  
  
  def looks_like_blank(line):
      line = line.split('//')[0]
      line = line.replace('{', '')
      line = line.replace('}', '')
      line = line.replace('[', '')
      line = line.replace(']', '')
      line = line.replace(';', '')
      line = line.strip()
      return len(line) == 0
  
  
  def looks_like_variable(line):
      line = line.split('//')[0]
      line = line.strip()
      if ' ' not in line and '\t' not in line:
          return False
      if line.startswith('friend '):
          return False
      if line.startswith('enum '):
          return False
      if line.startswith(','):
          return False
      if line.startswith(':'):
          return False
      if line.startswith('typedef'):
          return False
      if line.startswith('using'):
          return False
      if ' = ' in line:
          return True
      if line.endswith(';'):
          return True
      return False
  
  
  def looks_like_constant(line):
      line = line.strip()
      if line.startswith('inline'):
          line = line.split('inline')[1]
      line = line.strip()
      if not line.startswith('constexpr'):
          return False
      line = line.split('constexpr')[1]
      return looks_like_variable(line)
  
  
  def looks_like_forward_decl(line):
      line = line.split('//')[0]
      line = line.strip()
      if not line.endswith(';'):
          return False
      if '{' in line:
          return False
      if '}' in line:
          return False
      if line.startswith('friend '):
          return True
      if line.startswith('struct '):
          return True
      if line.startswith('class '):
          return True
      return False
  
  
  def looks_like_function(line):
      line = line.split('//')[0]
      if line.startswith('friend class '):
          return False
      if line.startswith('friend struct '):
          return False
      if '::' in line.split('(')[0].split(' ')[-1]:
          return False
      if line.startswith(','):
          return False
      if line.startswith(':'):
          return False
      return '(' in line
  
  
  def parse_function(lno, lines, filename):
  
      start_paren = 0
      end_paren = 0
      signature = ''
  
      global orphaned_export
      orphaned_export = False
  
      while lno < len(lines):
          line = lines[lno].strip()
          lno += 1
          if line.startswith('//'):
              continue
  
          start_paren += line.count('(')
          end_paren += line.count(')')
  
          sig_line = line.replace('TORRENT_EXPORT ', '') \
              .replace('TORRENT_EXTRA_EXPORT', '') \
              .replace('TORRENT_V3_EXPLICIT', '') \
              .replace('TORRENT_COUNTER_NOEXCEPT', '') \
              .split('//')[0].strip()
          if signature != '':
              sig_line = '\n   ' + sig_line
          signature += sig_line
          if verbose:
              print('fun     %s' % line)
  
          if start_paren > 0 and start_paren == end_paren:
              if signature[-1] != ';':
                  # we also need to consume the function body
                  start_paren = 0
                  end_paren = 0
                  for i in range(len(signature)):
                      if signature[i] == '(':
                          start_paren += 1
                      elif signature[i] == ')':
                          end_paren += 1
  
                      if start_paren > 0 and start_paren == end_paren:
                          for k in range(i, len(signature)):
                              if signature[k] == ':' or signature[k] == '{':
                                  signature = signature[0:k].strip()
                                  break
                          break
  
                  lno = consume_block(lno - 1, lines)
                  signature += ';'
              ret = [{'file': filename[11:], 'signatures': set([signature]), 'names': set(
                  [signature.split('(')[0].split(' ')[-1].strip() + '()'])}, lno]
              if first_item(ret[0]['names']) == '()':
                  return [None, lno]
              return ret
      if len(signature) > 0:
          print('\x1b[31mFAILED TO PARSE FUNCTION\x1b[0m %s\nline: %d\nfile: %s' % (signature, lno, filename))
      return [None, lno]
  
  
  def add_desc(line):
      # plain output prints just descriptions and filters out c++ code.
      # it's used to run spell checker over
      if plain_output:
          for s in line.split('\n'):
              # if the first character is a space, strip it
              if len(s) > 0 and s[0] == ' ':
                  s = s[1:]
              global in_code
              if in_code is not None and not s.startswith(in_code) and len(s) > 1:
                  in_code = None
  
              if s.strip().startswith('.. code::'):
                  in_code = s.split('.. code::')[0] + '\t'
  
              # strip out C++ code from the plain text output since it's meant for
              # running spell checking over
              if not s.strip().startswith('.. ') and in_code is None:
                  plain_file.write(s + '\n')
  
  
  def parse_class(lno, lines, filename):
      start_brace = 0
      end_brace = 0
  
      name = ''
      funs = []
      fields = []
      enums = []
      state = 'public'
      context = ''
      class_type = 'struct'
      blanks = 0
      decl = ''
  
      while lno < len(lines):
          line = lines[lno].strip()
          decl += lines[lno].replace('TORRENT_EXPORT ', '') \
              .replace('TORRENT_EXTRA_EXPORT', '') \
              .replace('TORRENT_V3_EXPLICIT', '') \
              .replace('TORRENT_COUNTER_NOEXCEPT', '').split('{')[0].strip()
          if '{' in line:
              break
          if verbose:
              print('class  %s' % line)
          lno += 1
  
      if decl.startswith('class'):
          state = 'private'
          class_type = 'class'
  
      name = decl.split(':')[0].replace('class ', '').replace('struct ', '').replace('final', '').strip()
  
      global orphaned_export
      orphaned_export = False
  
      while lno < len(lines):
          line = lines[lno].strip()
          lno += 1
  
          if line == '':
              blanks += 1
              context = ''
              continue
  
          if line.startswith('/*'):
              lno = consume_comment(lno - 1, lines)
              continue
  
          if line.startswith('#'):
              lno = consume_ifdef(lno - 1, lines, True)
              continue
  
          if 'TORRENT_DEFINE_ALERT' in line:
              if verbose:
                  print('xx    %s' % line)
              blanks += 1
              continue
          if 'TORRENT_DEPRECATED' in line:
              if verbose:
                  print('xx    %s' % line)
              blanks += 1
              continue
  
          if line.startswith('//'):
              if verbose:
                  print('desc  %s' % line)
  
              line = line[2:]
              if len(line) and line[0] == ' ':
                  line = line[1:]
              context += line + '\n'
              continue
  
          start_brace += line.count('{')
          end_brace += line.count('}')
  
          if line == 'private:':
              state = 'private'
          elif line == 'protected:':
              state = 'protected'
          elif line == 'public:':
              state = 'public'
  
          if start_brace > 0 and start_brace == end_brace:
              return [{'file': filename[11:], 'enums': enums, 'fields':fields,
                       'type': class_type, 'name': name, 'decl': decl, 'fun': funs}, lno]
  
          if state != 'public' and not internal:
              if verbose:
                  print('private %s' % line)
              blanks += 1
              continue
  
          if start_brace - end_brace > 1:
              if verbose:
                  print('scope   %s' % line)
              blanks += 1
              continue
  
          if looks_like_function(line):
              current_fun, lno = parse_function(lno - 1, lines, filename)
              if current_fun is not None and is_visible(context):
                  if context == '' and blanks == 0 and len(funs):
                      funs[-1]['signatures'].update(current_fun['signatures'])
                      funs[-1]['names'].update(current_fun['names'])
                  else:
                      if 'TODO: ' in context:
                          print('TODO comment in public documentation: %s:%d' % (filename, lno))
                          sys.exit(1)
                      current_fun['desc'] = context
                      add_desc(context)
                      if context == '' and not suppress_warning(filename, first_item(current_fun['names'])):
                          print('WARNING: member function "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                                % (name + '::' + first_item(current_fun['names']), filename, lno))
                      funs.append(current_fun)
                  context = ''
                  blanks = 0
              continue
  
          if looks_like_variable(line):
              if 'constexpr static' in line:
                  print('ERROR: found "constexpr static", use "static constexpr" instead for consistency!\n%s:%d\n%s'
                        % (filename, lno, line))
                  sys.exit(1)
              if verbose:
                  print('var     %s' % line)
              if not is_visible(context):
                  continue
              line = line.split('//')[0].strip()
              # the name may look like this:
              # std::uint8_t fails : 7;
              # int scrape_downloaded = -1;
              # static constexpr peer_flags_t interesting{0x1};
              n = line.split('=')[0].split('{')[0].strip().split(' : ')[0].split(' ')[-1].split(':')[0].split(';')[0]
              if context == '' and blanks == 0 and len(fields):
                  fields[-1]['names'].append(n)
                  fields[-1]['signatures'].append(line)
              else:
                  if context == '' and not suppress_warning(filename, n):
                      print('WARNING: field "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                            % (name + '::' + n, filename, lno))
                  add_desc(context)
                  fields.append({'signatures': [line], 'names': [n], 'desc': context})
              context = ''
              blanks = 0
              continue
  
          if line.startswith('enum '):
              if verbose:
                  print('enum    %s' % line)
              if not is_visible(context):
                  consume_block(lno - 1, lines)
              else:
                  enum, lno = parse_enum(lno - 1, lines, filename)
                  if enum is not None:
                      if 'TODO: ' in context:
                          print('TODO comment in public documentation: %s:%d' % (filename, lno))
                          sys.exit(1)
                      enum['desc'] = context
                      add_desc(context)
                      if context == '' and not suppress_warning(filename, enum['name']):
                          print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                                % (name + '::' + enum['name'], filename, lno))
                      enums.append(enum)
                  context = ''
              continue
  
          context = ''
  
          if verbose:
              if looks_like_forward_decl(line) \
                      or looks_like_blank(line) \
                      or looks_like_namespace(line):
                  print('--      %s' % line)
              else:
                  print('??      %s' % line)
  
      if len(name) > 0:
          print('\x1b[31mFAILED TO PARSE CLASS\x1b[0m %s\nfile: %s:%d' % (name, filename, lno))
      return [None, lno]
  
  
  def parse_constant(lno, lines, filename):
      line = lines[lno].strip()
      if verbose:
          print('const   %s' % line)
      line = line.split('=')[0]
      if 'constexpr' in line:
          line = line.split('constexpr')[1]
      if '{' in line and '}' in line:
          line = line.split('{')[0]
      t, name = line.strip().rsplit(' ', 1)
      return [{'file': filename[11:], 'type': t, 'name': name}, lno + 1]
  
  
  def parse_enum(lno, lines, filename):
      start_brace = 0
      end_brace = 0
      global anon_index
  
      line = lines[lno].strip()
      name = line.replace('enum ', '').replace('class ', '').split(':')[0].split('{')[0].strip()
      if len(name) == 0:
          if not internal:
              print('WARNING: anonymous enum at: \x1b[34m%s:%d\x1b[0m' % (filename, lno))
              lno = consume_block(lno - 1, lines)
              return [None, lno]
          name = 'anonymous_enum_%d' % anon_index
          anon_index += 1
  
      values = []
      context = ''
      if '{' not in line:
          if verbose:
              print('enum  %s' % lines[lno])
          lno += 1
  
      val = 0
      while lno < len(lines):
          line = lines[lno].strip()
          lno += 1
  
          if line.startswith('//'):
              if verbose:
                  print('desc  %s' % line)
              line = line[2:]
              if len(line) and line[0] == ' ':
                  line = line[1:]
              context += line + '\n'
              continue
  
          if line.startswith('#'):
              lno = consume_ifdef(lno - 1, lines)
              continue
  
          start_brace += line.count('{')
          end_brace += line.count('}')
  
          if '{' in line:
              line = line.split('{')[1]
          line = line.split('}')[0]
  
          if len(line):
              if verbose:
                  print('enumv %s' % lines[lno - 1])
              for v in line.split(','):
                  v = v.strip()
                  if v.startswith('//'):
                      break
                  if v == '':
                      continue
                  valstr = ''
                  try:
                      if '=' in v:
                          val = int(v.split('=')[1].strip(), 0)
                      valstr = str(val)
                  except Exception:
                      pass
  
                  if '=' in v:
                      v = v.split('=')[0].strip()
                  if is_visible(context):
                      add_desc(context)
                      values.append({'name': v.strip(), 'desc': context, 'val': valstr})
                      if verbose:
                          print('enumv %s' % valstr)
                  context = ''
                  val += 1
          else:
              if verbose:
                  print('??    %s' % lines[lno - 1])
  
          if start_brace > 0 and start_brace == end_brace:
              return [{'file': filename[11:], 'name': name, 'values': values}, lno]
  
      if len(name) > 0:
          print('\x1b[31mFAILED TO PARSE ENUM\x1b[0m %s\nline: %d\nfile: %s' % (name, lno, filename))
      return [None, lno]
  
  
  def consume_block(lno, lines):
      start_brace = 0
      end_brace = 0
  
      while lno < len(lines):
          line = lines[lno].strip()
          if verbose:
              print('xx    %s' % line)
          lno += 1
  
          start_brace += line.count('{')
          end_brace += line.count('}')
  
          if start_brace > 0 and start_brace == end_brace:
              break
      return lno
  
  
  def consume_comment(lno, lines):
      while lno < len(lines):
          line = lines[lno].strip()
          if verbose:
              print('xx    %s' % line)
          lno += 1
          if '*/' in line:
              break
  
      return lno
  
  
  def trim_define(line):
      return line.replace('#ifndef', '').replace('#ifdef', '') \
              .replace('#if', '').replace('defined', '') \
              .replace('TORRENT_ABI_VERSION == 1', '') \
              .replace('TORRENT_ABI_VERSION <= 2', '') \
              .replace('TORRENT_ABI_VERSION < 3', '') \
              .replace('||', '').replace('&&', '').replace('(', '').replace(')', '') \
              .replace('!', '').replace('\\', '').strip()
  
  
  def consume_ifdef(lno, lines, warn_on_ifdefs=False):
      line = lines[lno].strip()
      lno += 1
  
      start_if = 1
      end_if = 0
  
      if verbose:
          print('prep  %s' % line)
  
      if warn_on_ifdefs and line.strip().startswith('#if'):
          while line.endswith('\\'):
              lno += 1
              line += lines[lno].strip()
              if verbose:
                  print('prep  %s' % lines[lno].trim())
          define = trim_define(line)
          if 'TORRENT_' in define and 'TORRENT_ABI_VERSION' not in define:
              print('\x1b[31mWARNING: possible ABI breakage in public struct! "%s" \x1b[34m %s:%d\x1b[0m' %
                    (define, filename, lno))
          elif define != '':
              print('\x1b[33msensitive define in public struct: "%s"\x1b[34m %s:%d\x1b[0m' % (define, filename, lno))
  
      if (line.startswith('#if') and (
              ' TORRENT_USE_ASSERTS' in line or
              ' TORRENT_USE_INVARIANT_CHECKS' in line or
              ' TORRENT_ASIO_DEBUGGING' in line) or
              line == '#if TORRENT_ABI_VERSION == 1' or
              line == '#if TORRENT_ABI_VERSION <= 2' or
              line == '#if TORRENT_ABI_VERSION < 3'):
          while lno < len(lines):
              line = lines[lno].strip()
              lno += 1
              if verbose:
                  print('prep  %s' % line)
              if line.startswith('#endif'):
                  end_if += 1
              if line.startswith('#if'):
                  start_if += 1
              if line == '#else' and start_if - end_if == 1:
                  break
              if start_if - end_if == 0:
                  break
          return lno
      else:
          while line.endswith('\\') and lno < len(lines):
              line = lines[lno].strip()
              lno += 1
              if verbose:
                  print('prep  %s' % line)
  
      return lno
  
  
  for filename in files:
      h = open(filename)
      lines = h.read().split('\n')
  
      if verbose:
          print('\n=== %s ===\n' % filename)
  
      blanks = 0
      lno = 0
      orphaned_export = False
  
      while lno < len(lines):
          line = lines[lno].strip()
  
          if orphaned_export:
              print('ERROR: TORRENT_EXPORT without function or class!\n%s:%d\n%s' % (filename, lno, line))
              sys.exit(1)
  
          lno += 1
  
          if line == '':
              blanks += 1
              context = ''
              continue
  
          if 'TORRENT_EXPORT' in line.split() \
                  and 'ifndef TORRENT_EXPORT' not in line \
                  and 'define TORRENT_DEPRECATED_EXPORT TORRENT_EXPORT' not in line \
                  and 'define TORRENT_EXPORT' not in line \
                  and 'for TORRENT_EXPORT' not in line \
                  and 'TORRENT_EXPORT TORRENT_CFG' not in line \
                  and 'extern TORRENT_EXPORT ' not in line \
                  and 'struct TORRENT_EXPORT ' not in line:
              orphaned_export = True
              if verbose:
                  print('maybe orphaned: %s\n' % line)
  
          if line.startswith('//') and line[2:].strip() == 'OVERVIEW':
              # this is a section overview
              current_overview = ''
              while lno < len(lines):
                  line = lines[lno].strip()
                  lno += 1
                  if not line.startswith('//'):
                      # end of overview
                      overviews[filename[11:]] = current_overview
                      current_overview = ''
                      break
                  line = line[2:]
                  if line.startswith(' '):
                      line = line[1:]
                  current_overview += line + '\n'
  
          if line.startswith('//'):
              if verbose:
                  print('desc  %s' % line)
              line = line[2:]
              if len(line) and line[0] == ' ':
                  line = line[1:]
              context += line + '\n'
              continue
  
          if line.startswith('/*'):
              lno = consume_comment(lno - 1, lines)
              continue
  
          if line.startswith('#'):
              lno = consume_ifdef(lno - 1, lines)
              continue
  
          if (line == 'namespace aux {' or
                  line == 'namespace ssl {' or
                  line == 'namespace libtorrent { namespace aux {') \
                  and not internal:
              lno = consume_block(lno - 1, lines)
              context = ''
              continue
  
          if 'namespace aux' in line and \
                  '//' not in line.split('namespace')[0] and \
                  '}' not in line.split('namespace')[1]:
              print('ERROR: whitespace preceding namespace declaration: %s:%d' % (filename, lno))
              sys.exit(1)
  
          if 'TORRENT_DEPRECATED' in line:
              if ('class ' in line or 'struct ' in line) and ';' not in line:
                  lno = consume_block(lno - 1, lines)
                  context = ''
              blanks += 1
              if verbose:
                  print('xx    %s' % line)
              continue
  
          if looks_like_constant(line):
              if 'constexpr static' in line:
                  print('ERROR: found "constexpr static", use "static constexpr" instead for consistency!\n%s:%d\n%s'
                        % (filename, lno, line))
                  sys.exit(1)
              current_constant, lno = parse_constant(lno - 1, lines, filename)
              if current_constant is not None and is_visible(context):
                  if 'TODO: ' in context:
                      print('TODO comment in public documentation: %s:%d' % (filename, lno))
                      sys.exit(1)
                  current_constant['desc'] = context
                  add_desc(context)
                  if context == '':
                      print('WARNING: constant "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                            % (current_constant['name'], filename, lno))
                  t = current_constant['type']
                  if t in constants:
                      constants[t].append(current_constant)
                  else:
                      constants[t] = [current_constant]
              continue
  
          if 'TORRENT_EXPORT ' in line or line.startswith('inline ') or line.startswith('template') or internal:
              if line.startswith('class ') or line.startswith('struct '):
                  if not line.endswith(';'):
                      current_class, lno = parse_class(lno - 1, lines, filename)
                      if current_class is not None and is_visible(context):
                          if 'TODO: ' in context:
                              print('TODO comment in public documentation: %s:%d' % (filename, lno))
                              sys.exit(1)
                          current_class['desc'] = context
                          add_desc(context)
                          if context == '':
                              print('WARNING: class "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                                    % (current_class['name'], filename, lno))
                          classes.append(current_class)
                  context = ''
                  blanks += 1
                  continue
  
              if looks_like_function(line):
                  current_fun, lno = parse_function(lno - 1, lines, filename)
                  if current_fun is not None and is_visible(context):
                      if context == '' and blanks == 0 and len(functions):
                          functions[-1]['signatures'].update(current_fun['signatures'])
                          functions[-1]['names'].update(current_fun['names'])
                      else:
                          if 'TODO: ' in context:
                              print('TODO comment in public documentation: %s:%d' % (filename, lno))
                              sys.exit(1)
                          current_fun['desc'] = context
                          add_desc(context)
                          if context == '':
                              print('WARNING: function "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                                    % (first_item(current_fun['names']), filename, lno))
                          functions.append(current_fun)
                      context = ''
                      blanks = 0
                  continue
  
          if ('enum class ' not in line and 'class ' in line or 'struct ' in line) and ';' not in line:
              lno = consume_block(lno - 1, lines)
              context = ''
              blanks += 1
              continue
  
          if line.startswith('enum '):
              if not is_visible(context):
                  consume_block(lno - 1, lines)
              else:
                  current_enum, lno = parse_enum(lno - 1, lines, filename)
                  if current_enum is not None and is_visible(context):
                      if 'TODO: ' in context:
                          print('TODO comment in public documentation: %s:%d' % (filename, lno))
                          sys.exit(1)
                      current_enum['desc'] = context
                      add_desc(context)
                      if context == '':
                          print('WARNING: enum "%s" is not documented: \x1b[34m%s:%d\x1b[0m'
                                % (current_enum['name'], filename, lno))
                      enums.append(current_enum)
              context = ''
              blanks += 1
              continue
  
          blanks += 1
          if verbose:
              if looks_like_forward_decl(line) \
                      or looks_like_blank(line) \
                      or looks_like_namespace(line):
                  print('--    %s' % line)
              else:
                  print('??    %s' % line)
  
          context = ''
      h.close()
  
  # ====================================================================
  #
  #                               RENDER PART
  #
  # ====================================================================
  
  
  def new_category(cat):
      return {'classes': [], 'functions': [], 'enums': [],
              'filename': 'reference-%s.rst' % cat.replace(' ', '_'),
              'constants': {}}
  
  
  if dump:
  
      if verbose:
          print('\n===============================\n')
  
      for c in classes:
          print('\x1b[4m%s\x1b[0m %s\n{' % (c['type'], c['name']))
          for f in c['fun']:
              for s in f['signatures']:
                  print('   %s' % s.replace('\n', '\n   '))
  
          if len(c['fun']) > 0 and len(c['fields']) > 0:
              print('')
  
          for f in c['fields']:
              for s in f['signatures']:
                  print('   %s' % s)
  
          if len(c['fields']) > 0 and len(c['enums']) > 0:
              print('')
  
          for e in c['enums']:
              print('   \x1b[4menum\x1b[0m %s\n   {' % e['name'])
              for v in e['values']:
                  print('      %s' % v['name'])
              print('   };')
          print('};\n')
  
      for f in functions:
          print('%s' % f['signature'])
  
      for e in enums:
          print('\x1b[4menum\x1b[0m %s\n{' % e['name'])
          for v in e['values']:
              print('   %s' % v['name'])
          print('};')
  
      for t, c in constants:
          print('\x1b[4mconstant\x1b[0m %s %s\n' % (e['type'], e['name']))
  
  categories = {}
  
  for c in classes:
      cat = categorize_symbol(c['name'], c['file'])
      if cat not in categories:
          categories[cat] = new_category(cat)
  
      if c['file'] in overviews:
          categories[cat]['overview'] = overviews[c['file']]
  
      filename = categories[cat]['filename'].replace('.rst', '.html') + '#'
      categories[cat]['classes'].append(c)
      symbols[c['name']] = filename + c['name']
      for f in c['fun']:
          for n in f['names']:
              symbols[n] = filename + n
              symbols[c['name'] + '::' + n] = filename + n
  
      for f in c['fields']:
          for n in f['names']:
              symbols[c['name'] + '::' + n] = filename + n
  
      for e in c['enums']:
          symbols[e['name']] = filename + e['name']
          symbols[c['name'] + '::' + e['name']] = filename + e['name']
          for v in e['values']:
              # symbols[v['name']] = filename + v['name']
              symbols[e['name'] + '::' + v['name']] = filename + v['name']
              symbols[c['name'] + '::' + v['name']] = filename + v['name']
  
  for f in functions:
      cat = categorize_symbol(first_item(f['names']), f['file'])
      if cat not in categories:
          categories[cat] = new_category(cat)
  
      if f['file'] in overviews:
          categories[cat]['overview'] = overviews[f['file']]
  
      for n in f['names']:
          symbols[n] = categories[cat]['filename'].replace('.rst', '.html') + '#' + n
      categories[cat]['functions'].append(f)
  
  for e in enums:
      cat = categorize_symbol(e['name'], e['file'])
      if cat not in categories:
          categories[cat] = new_category(cat)
      categories[cat]['enums'].append(e)
      filename = categories[cat]['filename'].replace('.rst', '.html') + '#'
      symbols[e['name']] = filename + e['name']
      for v in e['values']:
          symbols[e['name'] + '::' + v['name']] = filename + v['name']
  
  for t, c in constants.items():
      for const in c:
          cat = categorize_symbol(t, const['file'])
          if cat not in categories:
              categories[cat] = new_category(cat)
          if t not in categories[cat]['constants']:
              categories[cat]['constants'][t] = [const]
          else:
              categories[cat]['constants'][t].append(const)
          filename = categories[cat]['filename'].replace('.rst', '.html') + '#'
          symbols[t + '::' + const['name']] = filename + t + '::' + const['name']
      symbols[t] = filename + t
  
  
  def print_declared_in(out, o):
      out.write('Declared in "%s"\n\n' % print_link(o['file'], 'include/%s' % o['file']))
      print(dump_link_targets(), file=out)
  
  # returns RST marked up string
  
  
  def linkify_symbols(string):
      lines = string.split('\n')
      ret = []
      in_literal = False
      lno = 0
      return_string = ''
      for line in lines:
          lno += 1
          # don't touch headlines, i.e. lines whose
          # next line entirely contains one of =, - or .
          if (lno < len(lines) - 1):
              next_line = lines[lno]
          else:
              next_line = ''
  
          if '.. include:: ' in line:
              return_string += '\n'.join(ret)
              ret = [line]
              return_string += dump_link_targets() + '\n'
              continue
  
          if len(next_line) > 0 and lines[lno].replace('=', ''). \
                  replace('-', '').replace('.', '') == '':
              ret.append(line)
              continue
  
          if line.startswith('|'):
              ret.append(line)
              continue
          if in_literal and not line.startswith('\t') and not line == '':
              # print('  end literal: "%s"' % line)
              in_literal = False
          if in_literal:
              # print('  literal: "%s"' % line)
              ret.append(line)
              continue
          if line.strip() == '.. parsed-literal::' or \
                  line.strip().startswith('.. code::') or \
                  (not line.strip().startswith('..') and line.endswith('::')):
              # print('  start literal: "%s"' % line)
              in_literal = True
          words = line.split(' ')
  
          for i in range(len(words)):
              # it's important to preserve leading
              # tabs, since that's relevant for
              # rst markup
  
              leading = ''
              w = words[i]
  
              if len(w) == 0:
                  continue
  
              while len(w) > 0 and \
                      w[0] in ['\t', ' ', '(', '[', '{']:
                  leading += w[0]
                  w = w[1:]
  
              # preserve commas and dots at the end
              w = w.strip()
              trailing = ''
  
              if len(w) == 0:
                  continue
  
              while len(w) > 1 and w[-1] in ['.', ',', ')'] and w[-2:] != '()':
                  trailing = w[-1] + trailing
                  w = w[:-1]
  
              link_name = w
  
              # print(w)
  
              if len(w) == 0:
                  continue
  
              if link_name[-1] == '_':
                  link_name = link_name[:-1]
  
              if w in symbols:
                  link_name = link_name.replace('-', ' ')
                  # print('  found %s -> %s' % (w, link_name))
                  words[i] = leading + print_link(link_name, symbols[w]) + trailing
          ret.append(' '.join(words))
      return_string += '\n'.join(ret)
      return return_string
  
  
  link_targets = []
  
  
  def print_link(name, target):
      global link_targets
      link_targets.append(target)
      return "`%s`__" % name
  
  
  def dump_link_targets(indent=''):
      global link_targets
      ret = '\n'
      for link in link_targets:
          ret += '%s__ %s\n' % (indent, link)
      link_targets = []
      return ret
  
  
  def heading(string, c, indent=''):
      string = string.strip()
      return '\n' + indent + string + '\n' + indent + (c * len(string)) + '\n'
  
  
  def render_enums(out, enums, print_declared_reference, header_level):
      for e in enums:
          print('.. raw:: html\n', file=out)
          print('\t<a name="%s"></a>' % e['name'], file=out)
          print('', file=out)
          dump_report_issue('enum ' + e['name'], out)
          print(heading('enum %s' % e['name'], header_level), file=out)
  
          print_declared_in(out, e)
  
          width = [len('name'), len('value'), len('description')]
  
          for i in range(len(e['values'])):
              e['values'][i]['desc'] = linkify_symbols(e['values'][i]['desc'])
  
          for v in e['values']:
              width[0] = max(width[0], len(v['name']))
              width[1] = max(width[1], len(v['val']))
              for d in v['desc'].split('\n'):
                  width[2] = max(width[2], len(d))
  
          print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out)
          print('| ' + 'name'.ljust(width[0]) + ' | ' + 'value'.ljust(width[1]) + ' | '
                + 'description'.ljust(width[2]) + ' |', file=out)
          print('+=' + ('=' * width[0]) + '=+=' + ('=' * width[1]) + '=+=' + ('=' * width[2]) + '=+', file=out)
          for v in e['values']:
              d = v['desc'].split('\n')
              if len(d) == 0:
                  d = ['']
              print('| ' + v['name'].ljust(width[0]) + ' | ' + v['val'].ljust(width[1]) + ' | '
                    + d[0].ljust(width[2]) + ' |', file=out)
              for s in d[1:]:
                  print('| ' + (' ' * width[0]) + ' | ' + (' ' * width[1]) + ' | ' + s.ljust(width[2]) + ' |', file=out)
              print('+-' + ('-' * width[0]) + '-+-' + ('-' * width[1]) + '-+-' + ('-' * width[2]) + '-+', file=out)
          print('', file=out)
  
          print(dump_link_targets(), file=out)
  
  
  sections = \
      {
          'Core': 0,
          'DHT': 0,
          'Session': 0,
          'Torrent Handle': 0,
          'Torrent Info': 0,
          'Trackers': 0,
          'Settings': 0,
          'Torrent Status': 0,
          'Stats': 0,
          'Resume Data': 0,
          'Add Torrent': 0,
  
          'Bencoding': 1,
          'Bdecoding': 1,
          'Filter': 1,
          'Error Codes': 1,
          'Create Torrents': 1,
  
          'PeerClass': 2,
          'ed25519': 2,
          'Utility': 2,
          'Storage': 2,
          'Custom Storage': 2,
          'Plugins': 2,
  
          'Alerts': 3
      }
  
  
  def print_toc(out, categories, s):
  
      main_toc = False
  
      for cat in categories:
          if (s != 2 and cat not in sections) or \
                  (cat in sections and sections[cat] != s):
              continue
  
          if not main_toc:
              out.write('.. container:: main-toc\n\n')
              main_toc = True
  
          print('\t.. rubric:: %s\n' % cat, file=out)
  
          if 'overview' in categories[cat]:
              print('\t| overview__', file=out)
  
          for c in categories[cat]['classes']:
              print('\t| ' + print_link(c['name'], symbols[c['name']]), file=out)
          for f in categories[cat]['functions']:
              for n in f['names']:
                  print('\t| ' + print_link(n, symbols[n]), file=out)
          for e in categories[cat]['enums']:
              print('\t| ' + print_link(e['name'], symbols[e['name']]), file=out)
          for t, c in categories[cat]['constants'].items():
              print('\t| ' + print_link(t, symbols[t]), file=out)
          print('', file=out)
  
          if 'overview' in categories[cat]:
              print('\t__ %s#overview' % categories[cat]['filename'].replace('.rst', '.html'), file=out)
          print(dump_link_targets('\t'), file=out)
  
  
  def dump_report_issue(h, out):
      print(('.. raw:: html\n\n\t<span class="report-issue">[<a ' +
             'href="http://github.com/arvidn/libtorrent/issues/new?title=docs:{0}&labels=' +
             'documentation&body={1}">report issue</a>]</span>\n\n').format(
                  urllib.parse.quote_plus(h),
                  urllib.parse.quote_plus('Documentation under heading "' + h + '" could be improved')), file=out)
  
  
  def render(out, category):
  
      classes = category['classes']
      functions = category['functions']
      enums = category['enums']
      constants = category['constants']
  
      if 'overview' in category:
          out.write('%s\n' % linkify_symbols(category['overview']))
  
      for c in classes:
  
          print('.. raw:: html\n', file=out)
          print('\t<a name="%s"></a>' % c['name'], file=out)
          print('', file=out)
  
          dump_report_issue('class ' + c['name'], out)
          out.write('%s\n' % heading(c['name'], '-'))
          print_declared_in(out, c)
          c['desc'] = linkify_symbols(c['desc'])
          out.write('%s\n' % c['desc'])
          print(dump_link_targets(), file=out)
  
          print('\n.. parsed-literal::\n\t', file=out)
  
          block = '\n%s\n{\n' % c['decl']
          for f in c['fun']:
              for s in f['signatures']:
                  block += '   %s\n' % highlight_signature(s.replace('\n', '\n   '))
  
          if len(c['fun']) > 0 and len(c['enums']) > 0:
              block += '\n'
  
          first = True
          for e in c['enums']:
              if not first:
                  block += '\n'
              first = False
              block += '   enum %s\n   {\n' % e['name']
              for v in e['values']:
                  block += '      %s,\n' % v['name']
              block += '   };\n'
  
          if len(c['fun']) + len(c['enums']) > 0 and len(c['fields']):
              block += '\n'
  
          for f in c['fields']:
              for s in f['signatures']:
                  block += '   %s\n' % highlight_name(s)
  
          block += '};'
  
          print(block.replace('\n', '\n\t') + '\n', file=out)
  
          for f in c['fun']:
              if f['desc'] == '':
                  continue
              print('.. raw:: html\n', file=out)
              for n in f['names']:
                  print('\t<a name="%s"></a>' % n, file=out)
              print('', file=out)
              h = ' '.join(f['names'])
              dump_report_issue('%s::[%s]' % (c['name'], h), out)
              print(heading(h, '.'), file=out)
  
              block = '.. parsed-literal::\n\n'
  
              for s in f['signatures']:
                  block += highlight_signature(s.replace('\n', '\n   ')) + '\n'
              print('%s\n' % block.replace('\n', '\n\t'), file=out)
              f['desc'] = linkify_symbols(f['desc'])
              print('%s' % f['desc'], file=out)
  
              print(dump_link_targets(), file=out)
  
          render_enums(out, c['enums'], False, '.')
  
          for f in c['fields']:
              if f['desc'] == '':
                  continue
  
              print('.. raw:: html\n', file=out)
              for n in f['names']:
                  print('\t<a name="%s"></a>' % n, file=out)
              print('', file=out)
              h = ' '.join(f['names'])
              dump_report_issue('%s::[%s]' % (c['name'], h), out)
              print(h, file=out)
              f['desc'] = linkify_symbols(f['desc'])
              print('\t%s' % f['desc'].replace('\n', '\n\t'), file=out)
  
              print(dump_link_targets(), file=out)
  
      for f in functions:
          print('.. raw:: html\n', file=out)
          for n in f['names']:
              print('\t<a name="%s"></a>' % n, file=out)
          print('', file=out)
          h = ' '.join(f['names'])
          dump_report_issue(h, out)
          print(heading(h, '-'), file=out)
          print_declared_in(out, f)
  
          block = '.. parsed-literal::\n\n'
          for s in f['signatures']:
              block += highlight_signature(s) + '\n'
  
          print('%s\n' % block.replace('\n', '\n\t'), file=out)
          print(linkify_symbols(f['desc']), file=out)
  
          print(dump_link_targets(), file=out)
  
      render_enums(out, enums, True, '-')
  
      for t, c in constants.items():
          print('.. raw:: html\n', file=out)
          print('\t<a name="%s"></a>\n' % t, file=out)
          dump_report_issue(t, out)
          print(heading(t, '-'), file=out)
          print_declared_in(out, c[0])
  
          for v in c:
              print('.. raw:: html\n', file=out)
              print('\t<a name="%s::%s"></a>\n' % (t, v['name']), file=out)
              print(v['name'], file=out)
              v['desc'] = linkify_symbols(v['desc'])
              print('\t%s' % v['desc'].replace('\n', '\n\t'), file=out)
              print(dump_link_targets('\t'), file=out)
  
          print('', file=out)
  
      print(dump_link_targets(), file=out)
  
      for i in static_links:
          print(i, file=out)
  
  
  if single_page_output:
  
      out = open('single-page-ref.rst', 'w+')
      out.write('''.. include:: header.rst
  
  `home`__
  
  __ reference.html
  
  .. contents:: Table of contents
    :depth: 2
    :backlinks: none
  
  ''')
  
      for cat in categories:
          render(out, categories[cat])
  
      out.close()
  
  else:
  
      out = open('reference.rst', 'w+')
      out.write('''=======================
  reference documentation
  =======================
  
  ''')
  
      out.write('`single-page version`__\n\n__ single-page-ref.html\n\n')
  
      for i in range(4):
  
          print_toc(out, categories, i)
  
      out.close()
  
      for cat in categories:
          out = open(categories[cat]['filename'], 'w+')
  
          out.write('''.. include:: header.rst
  
  `home`__
  
  __ reference.html
  
  .. contents:: Table of contents
    :depth: 2
    :backlinks: none
  
  ''')
  
          render(out, categories[cat])
  
          out.close()
  
  #       for s in symbols:
  #           print(s)
  
      for i, o in list(preprocess_rst.items()):
          f = open(i, 'r')
          out = open(o, 'w+')
          print('processing %s -> %s' % (i, o))
          link = linkify_symbols(f.read())
          print(link, end=' ', file=out)
  
          print(dump_link_targets(), file=out)
  
          out.close()
          f.close()
  