tools/symsrc/source_index.py
def UpdatePDB(pdb_filename, verbose=True, build_dir=None, toolchain_dir=None,
              follow_junctions=False):
  """Update a pdb file with source information."""
  dir_exclusion_list = { }

  if build_dir:
    # Excluding the build directory allows skipping the generated files, for
    # Chromium this makes the indexing ~10x faster.
    build_dir = (os.path.normpath(build_dir)).lower()
    for directory, _, _ in os.walk(build_dir):
      dir_exclusion_list[directory.lower()] = True
    dir_exclusion_list[build_dir.lower()] = True

  if toolchain_dir:
    # Exclude the directories from the toolchain as we don't have revision info
    # for them.
    toolchain_dir = (os.path.normpath(toolchain_dir)).lower()
    for directory, _, _ in os.walk(toolchain_dir):
      dir_exclusion_list[directory.lower()] = True
    dir_exclusion_list[toolchain_dir.lower()] = True

  # Writes the header of the source index stream.
  #
  # Here's the description of the variables used in the SRC_* macros (those
  # variables have to be defined for every source file that we want to index):
  #   var1: The file path.
  #   var2: The name of the file without its path.
  #   var3: The revision or the hash of this file's repository.
  #   var4: The URL to this file.
  #   var5: (optional) The python method to call to decode this file, e.g. for
  #       a base64 encoded file this value should be 'base64.b64decode'.
  lines = [
    'SRCSRV: ini ------------------------------------------------',
    'VERSION=1',
    'INDEXVERSION=2',
    'VERCTRL=Subversion',
    'DATETIME=%s' % time.asctime(),
    'SRCSRV: variables ------------------------------------------',
    'SRC_EXTRACT_TARGET_DIR=%targ%\\%fnbksl%(%var2%)\\%var3%',
    'SRC_EXTRACT_TARGET=%SRC_EXTRACT_TARGET_DIR%\\%fnfile%(%var1%)',
    'SRC_EXTRACT_CMD=cmd /c "mkdir "%SRC_EXTRACT_TARGET_DIR%" & python3 -c '
        '"import urllib.request, base64;'
        'url = \\\"%var4%\\\";'
        'u = urllib.request.urlopen(url);'
        'open(r\\\"%SRC_EXTRACT_TARGET%\\\", \\\"wb\\\").write(%var5%('
            'u.read()))"',
    'SRCSRVTRG=%SRC_EXTRACT_TARGET%',
    'SRCSRVCMD=%SRC_EXTRACT_CMD%',
    'SRCSRV: source files ---------------------------------------',
  ]

  if ReadSourceStream(pdb_filename, toolchain_dir):
    raise Exception("PDB already has source indexing information!")

  filelist = ExtractSourceFiles(pdb_filename, toolchain_dir)
  number_of_files = len(filelist)
  indexed_files_total = 0

  t_init = time.time()
  t1 = t_init
  while filelist:
    filename = next(iter(filelist))
    filedir = os.path.dirname(filename)
    if verbose:
      print("[%d / %d] Processing: %s" % (number_of_files - len(filelist),
                                          number_of_files, filename))

    # Print a message every 60 seconds to make sure that the process doesn't
    # time out.
    if time.time() - t1 > 60:
      t1 = time.time()
      print("Still working, %d / %d files have been processed." %
            (number_of_files - len(filelist), number_of_files))

    # This directory is in the exclusion listed, either because it's not part of
    # a repository, or from one we're not interested in indexing.
    if dir_exclusion_list.get(filedir, False):
      if verbose:
        print("  skipping, directory is excluded.")
      filelist.remove(filename)
      continue

    # Skip the files that don't exist on the current machine.
    if not os.path.exists(filename):
      filelist.remove(filename)
      continue

    # Try to index the current file and all the ones coming from the same
    # repository.
    indexed_files = IndexFilesFromRepo(
        filename, filelist, lines, follow_junctions)
    if not indexed_files:
      if not DirectoryIsPartOfPublicGitRepository(filedir):
        dir_exclusion_list[filedir] = True
        if verbose:
          print("Adding %s to the exclusion list." % filedir)
      filelist.remove(filename)
      continue

    indexed_files_total += indexed_files

    if verbose:
      print("  %d files have been indexed." % indexed_files)

  print('Indexing took %d seconds' % (time.time() - t_init))

  lines.append('SRCSRV: end ------------------------------------------------')

  WriteSourceStream(pdb_filename, '\r\n'.join(lines), toolchain_dir)

  if verbose:
    print("%d / %d files have been indexed." % (indexed_files_total,
                                                number_of_files))
Update a pdb file with source information.
Update a pdb file with source information.
Update a pdb file with source information .
def UpdatePDB(pdb_filename, verbose=True, build_dir=None, toolchain_dir=None,
              follow_junctions=False):
  dir_exclusion_list = { }
  if build_dir:
    build_dir = (os.path.normpath(build_dir)).lower()
    for directory, _, _ in os.walk(build_dir):
      dir_exclusion_list[directory.lower()] = True
    dir_exclusion_list[build_dir.lower()] = True
  if toolchain_dir:
    toolchain_dir = (os.path.normpath(toolchain_dir)).lower()
    for directory, _, _ in os.walk(toolchain_dir):
      dir_exclusion_list[directory.lower()] = True
    dir_exclusion_list[toolchain_dir.lower()] = True
   Here's the description of the variables used in the SRC_* macros (those
   variables have to be defined for every source file that we want to index):
     var1: The file path.
     var2: The name of the file without its path.
     var3: The revision or the hash of this file's repository.
     var4: The URL to this file.
     var5: (optional) The python method to call to decode this file, e.g. for
         a base64 encoded file this value should be 'base64.b64decode'.
  lines = [
    'SRCSRV: ini ------------------------------------------------',
    'VERSION=1',
    'INDEXVERSION=2',
    'VERCTRL=Subversion',
    'DATETIME=%s' % time.asctime(),
    'SRCSRV: variables ------------------------------------------',
    'SRC_EXTRACT_TARGET_DIR=%targ%\\%fnbksl%(%var2%)\\%var3%',
    'SRC_EXTRACT_TARGET=%SRC_EXTRACT_TARGET_DIR%\\%fnfile%(%var1%)',
    'SRC_EXTRACT_CMD=cmd /c "mkdir "%SRC_EXTRACT_TARGET_DIR%" & python3 -c '
        '"import urllib.request, base64;'
        'url = \\\"%var4%\\\";'
        'u = urllib.request.urlopen(url);'
        'open(r\\\"%SRC_EXTRACT_TARGET%\\\", \\\"wb\\\").write(%var5%('
            'u.read()))"',
    'SRCSRVTRG=%SRC_EXTRACT_TARGET%',
    'SRCSRVCMD=%SRC_EXTRACT_CMD%',
    'SRCSRV: source files ---------------------------------------',
  ]
  if ReadSourceStream(pdb_filename, toolchain_dir):
    raise Exception("PDB already has source indexing information!")
  filelist = ExtractSourceFiles(pdb_filename, toolchain_dir)
  number_of_files = len(filelist)
  indexed_files_total = 0
  t_init = time.time()
  t1 = t_init
  while filelist:
    filename = next(iter(filelist))
    filedir = os.path.dirname(filename)
    if verbose:
      print("[%d / %d] Processing: %s" % (number_of_files - len(filelist),
                                          number_of_files, filename))
     Print a message every 60 seconds to make sure that the process doesn't
     time out.
    if time.time() - t1 > 60:
      t1 = time.time()
      print("Still working, %d / %d files have been processed." %
            (number_of_files - len(filelist), number_of_files))
     This directory is in the exclusion listed, either because it's not part of
     a repository, or from one we're not interested in indexing.
    if dir_exclusion_list.get(filedir, False):
      if verbose:
        print("  skipping, directory is excluded.")
      filelist.remove(filename)
      continue
     Skip the files that don't exist on the current machine.
    if not os.path.exists(filename):
      filelist.remove(filename)
      continue
     Try to index the current file and all the ones coming from the same
     repository.
    indexed_files = IndexFilesFromRepo(
        filename, filelist, lines, follow_junctions)
    if not indexed_files:
      if not DirectoryIsPartOfPublicGitRepository(filedir):
        dir_exclusion_list[filedir] = True
        if verbose:
          print("Adding %s to the exclusion list." % filedir)
      filelist.remove(filename)
      continue
    indexed_files_total += indexed_files
    if verbose:
      print("  %d files have been indexed." % indexed_files)
  print('Indexing took %d seconds' % (time.time() - t_init))
  lines.append('SRCSRV: end ------------------------------------------------')
  WriteSourceStream(pdb_filename, '\r\n'.join(lines), toolchain_dir)
  if verbose:
    print("%d / %d files have been indexed." % (indexed_files_total,
                                                number_of_files))
def UpdatePDB ( pdb_filename , verbose = True , build_dir = None , toolchain_dir = None , follow_junctions = False ) : dir_exclusion_list = { } if build_dir : build_dir = ( os . path . normpath ( build_dir ) ) . lower ( ) for directory , _ , _ in os . walk ( build_dir ) : dir_exclusion_list [ directory . lower ( ) ] = True dir_exclusion_list [ build_dir . lower ( ) ] = True if toolchain_dir : toolchain_dir = ( os . path . normpath ( toolchain_dir ) ) . lower ( ) for directory , _ , _ in os . walk ( toolchain_dir ) : dir_exclusion_list [ directory . lower ( ) ] = True dir_exclusion_list [ toolchain_dir . lower ( ) ] = True lines = [ 'SRCSRV: ini ------------------------------------------------' , 'VERSION=1' , 'INDEXVERSION=2' , 'VERCTRL=Subversion' , 'DATETIME=%s' % time . asctime ( ) , 'SRCSRV: variables ------------------------------------------' , 'SRC_EXTRACT_TARGET_DIR=%targ%\\%fnbksl%(%var2%)\\%var3%' , 'SRC_EXTRACT_TARGET=%SRC_EXTRACT_TARGET_DIR%\\%fnfile%(%var1%)' , 'SRC_EXTRACT_CMD=cmd /c "mkdir "%SRC_EXTRACT_TARGET_DIR%" & python3 -c ' '"import urllib.request, base64;' 'url = \\\"%var4%\\\";' 'u = urllib.request.urlopen(url);' 'open(r\\\"%SRC_EXTRACT_TARGET%\\\", \\\"wb\\\").write(%var5%(' 'u.read()))"' , 'SRCSRVTRG=%SRC_EXTRACT_TARGET%' , 'SRCSRVCMD=%SRC_EXTRACT_CMD%' , 'SRCSRV: source files ---------------------------------------' , ] if ReadSourceStream ( pdb_filename , toolchain_dir ) : raise Exception ( "PDB already has source indexing information!" ) filelist = ExtractSourceFiles ( pdb_filename , toolchain_dir ) number_of_files = len ( filelist ) indexed_files_total = 0 t_init = time . time ( ) t1 = t_init while filelist : filename = next ( iter ( filelist ) ) filedir = os . path . dirname ( filename ) if verbose : print ( "[%d / %d] Processing: %s" % ( number_of_files - len ( filelist ) , number_of_files , filename ) ) if time . time ( ) - t1 > 60 : t1 = time . time ( ) print ( "Still working, %d / %d files have been processed." % ( number_of_files - len ( filelist ) , number_of_files ) ) if dir_exclusion_list . get ( filedir , False ) : if verbose : print ( "  skipping, directory is excluded." ) filelist . remove ( filename ) continue if not os . path . exists ( filename ) : filelist . remove ( filename ) continue indexed_files = IndexFilesFromRepo ( filename , filelist , lines , follow_junctions ) if not indexed_files : if not DirectoryIsPartOfPublicGitRepository ( filedir ) : dir_exclusion_list [ filedir ] = True if verbose : print ( "Adding %s to the exclusion list." % filedir ) filelist . remove ( filename ) continue indexed_files_total += indexed_files if verbose : print ( "  %d files have been indexed." % indexed_files ) print ( 'Indexing took %d seconds' % ( time . time ( ) - t_init ) ) lines . append ( 'SRCSRV: end ------------------------------------------------' ) WriteSourceStream ( pdb_filename , '\r\n' . join ( lines ) , toolchain_dir ) if verbose : print ( "%d / %d files have been indexed." % ( indexed_files_total , number_of_files ) )
Update a pdb file with source information.
Update a pdb file with source information .
"""Update a pdb file with source information.""" # Excluding the build directory allows skipping the generated files, for # Chromium this makes the indexing ~10x faster. # Exclude the directories from the toolchain as we don't have revision info # for them. # Writes the header of the source index stream. # # Here's the description of the variables used in the SRC_* macros (those # variables have to be defined for every source file that we want to index): #   var1: The file path. #   var2: The name of the file without its path. #   var3: The revision or the hash of this file's repository. #   var4: The URL to this file. #   var5: (optional) The python method to call to decode this file, e.g. for #       a base64 encoded file this value should be 'base64.b64decode'. # Print a message every 60 seconds to make sure that the process doesn't # time out. # This directory is in the exclusion listed, either because it's not part of # a repository, or from one we're not interested in indexing. # Skip the files that don't exist on the current machine. # Try to index the current file and all the ones coming from the same # repository.
pdb_filename
verbose
build_dir
toolchain_dir
follow_junctions