tools/bisect-builds.py
def ParseDirectoryIndex(self, last_known_rev):
    """Parses the Google Storage directory listing into a list of revision
    numbers."""

    def _GetMarkerForRev(revision):
      if self.is_asan:
        return '%s-%s/%s-%d.zip' % (
            self.GetASANPlatformDir(), self.build_type,
            self.GetASANBaseName(), revision)
      return '%s%d' % (self._listing_platform_dir, revision)

    def _FetchAndParse(url):
      """Fetches a URL and returns a 2-Tuple of ([revisions], next-marker). If
      next-marker is not None, then the listing is a partial listing and another
      fetch should be performed with next-marker being the marker= GET
      parameter."""
      handle = urllib.urlopen(url)
      document = ElementTree.parse(handle)

      # All nodes in the tree are namespaced. Get the root's tag name to extract
      # the namespace. Etree does namespaces as |{namespace}tag|.
      root_tag = document.getroot().tag
      end_ns_pos = root_tag.find('}')
      if end_ns_pos == -1:
        raise Exception('Could not locate end namespace for directory index')
      namespace = root_tag[:end_ns_pos + 1]

      # Find the prefix (_listing_platform_dir) and whether or not the list is
      # truncated.
      prefix_len = len(document.find(namespace + 'Prefix').text)
      next_marker = None
      is_truncated = document.find(namespace + 'IsTruncated')
      if is_truncated is not None and is_truncated.text.lower() == 'true':
        next_marker = document.find(namespace + 'NextMarker').text
      # Get a list of all the revisions.
      revisions = []
      githash_svn_dict = {}
      if self.is_asan:
        asan_regex = re.compile(r'.*%s-(\d+)\.zip$' % (self.GetASANBaseName()))
        # Non ASAN builds are in a <revision> directory. The ASAN builds are
        # flat
        all_prefixes = document.findall(namespace + 'Contents/' +
                                        namespace + 'Key')
        for prefix in all_prefixes:
          m = asan_regex.match(prefix.text)
          if m:
            try:
              revisions.append(int(m.group(1)))
            except ValueError:
              pass
      else:
        all_prefixes = document.findall(namespace + 'CommonPrefixes/' +
                                        namespace + 'Prefix')
        # The <Prefix> nodes have content of the form of
        # |_listing_platform_dir/revision/|. Strip off the platform dir and the
        # trailing slash to just have a number.
        for prefix in all_prefixes:
          revnum = prefix.text[prefix_len:-1]
          try:
            revnum = int(revnum)
            revisions.append(revnum)
          # Notes:
          # Ignore hash in chromium-browser-snapshots as they are invalid
          # Resulting in 404 error in fetching pages:
          # https://chromium.googlesource.com/chromium/src/+/[rev_hash]
          except ValueError:
            pass
      return (revisions, next_marker, githash_svn_dict)

    # Fetch the first list of revisions.
    if last_known_rev:
      revisions = []
      # Optimization: Start paging at the last known revision (local cache).
      next_marker = _GetMarkerForRev(last_known_rev)
      # Optimization: Stop paging at the last known revision (remote).
      last_change_rev = GetChromiumRevision(self, self.GetLastChangeURL())
      if last_known_rev == last_change_rev:
        return []
    else:
      (revisions, next_marker, new_dict) = _FetchAndParse(self.GetListingURL())
      self.githash_svn_dict.update(new_dict)
      last_change_rev = None

    # If the result list was truncated, refetch with the next marker. Do this
    # until an entire directory listing is done.
    while next_marker:
      sys.stdout.write('\rFetching revisions at marker %s' % next_marker)
      sys.stdout.flush()

      next_url = self.GetListingURL(next_marker)
      (new_revisions, next_marker, new_dict) = _FetchAndParse(next_url)
      revisions.extend(new_revisions)
      self.githash_svn_dict.update(new_dict)
      if last_change_rev and last_change_rev in new_revisions:
        break
    sys.stdout.write('\r')
    sys.stdout.flush()
    return revisions
Parses the Google Storage directory listing into a list of revision
    numbers.
Parses the Google Storage directory listing into a list of revision
numbers.
Parses the Google Storage directory listing into a list of revision numbers .
def ParseDirectoryIndex(self, last_known_rev):
    def _GetMarkerForRev(revision):
      if self.is_asan:
        return '%s-%s/%s-%d.zip' % (
            self.GetASANPlatformDir(), self.build_type,
            self.GetASANBaseName(), revision)
      return '%s%d' % (self._listing_platform_dir, revision)
    def _FetchAndParse(url):
      handle = urllib.urlopen(url)
      document = ElementTree.parse(handle)
      root_tag = document.getroot().tag
      end_ns_pos = root_tag.find('}')
      if end_ns_pos == -1:
        raise Exception('Could not locate end namespace for directory index')
      namespace = root_tag[:end_ns_pos + 1]
      prefix_len = len(document.find(namespace + 'Prefix').text)
      next_marker = None
      is_truncated = document.find(namespace + 'IsTruncated')
      if is_truncated is not None and is_truncated.text.lower() == 'true':
        next_marker = document.find(namespace + 'NextMarker').text
      revisions = []
      githash_svn_dict = {}
      if self.is_asan:
        asan_regex = re.compile(r'.*%s-(\d+)\.zip$' % (self.GetASANBaseName()))
        all_prefixes = document.findall(namespace + 'Contents/' +
                                        namespace + 'Key')
        for prefix in all_prefixes:
          m = asan_regex.match(prefix.text)
          if m:
            try:
              revisions.append(int(m.group(1)))
            except ValueError:
              pass
      else:
        all_prefixes = document.findall(namespace + 'CommonPrefixes/' +
                                        namespace + 'Prefix')
        for prefix in all_prefixes:
          revnum = prefix.text[prefix_len:-1]
          try:
            revnum = int(revnum)
            revisions.append(revnum)
          except ValueError:
            pass
      return (revisions, next_marker, githash_svn_dict)
    if last_known_rev:
      revisions = []
      next_marker = _GetMarkerForRev(last_known_rev)
      last_change_rev = GetChromiumRevision(self, self.GetLastChangeURL())
      if last_known_rev == last_change_rev:
        return []
    else:
      (revisions, next_marker, new_dict) = _FetchAndParse(self.GetListingURL())
      self.githash_svn_dict.update(new_dict)
      last_change_rev = None
    while next_marker:
      sys.stdout.write('\rFetching revisions at marker %s' % next_marker)
      sys.stdout.flush()
      next_url = self.GetListingURL(next_marker)
      (new_revisions, next_marker, new_dict) = _FetchAndParse(next_url)
      revisions.extend(new_revisions)
      self.githash_svn_dict.update(new_dict)
      if last_change_rev and last_change_rev in new_revisions:
        break
    sys.stdout.write('\r')
    sys.stdout.flush()
    return revisions
def ParseDirectoryIndex ( self , last_known_rev ) : def _GetMarkerForRev ( revision ) : if self . is_asan : return '%s-%s/%s-%d.zip' % ( self . GetASANPlatformDir ( ) , self . build_type , self . GetASANBaseName ( ) , revision ) return '%s%d' % ( self . _listing_platform_dir , revision ) def _FetchAndParse ( url ) : """Fetches a URL and returns a 2-Tuple of ([revisions], next-marker). If
      next-marker is not None, then the listing is a partial listing and another
      fetch should be performed with next-marker being the marker= GET
      parameter.""" handle = urllib . urlopen ( url ) document = ElementTree . parse ( handle ) root_tag = document . getroot ( ) . tag end_ns_pos = root_tag . find ( '}' ) if end_ns_pos == - 1 : raise Exception ( 'Could not locate end namespace for directory index' ) namespace = root_tag [ : end_ns_pos + 1 ] prefix_len = len ( document . find ( namespace + 'Prefix' ) . text ) next_marker = None is_truncated = document . find ( namespace + 'IsTruncated' ) if is_truncated is not None and is_truncated . text . lower ( ) == 'true' : next_marker = document . find ( namespace + 'NextMarker' ) . text revisions = [ ] githash_svn_dict = { } if self . is_asan : asan_regex = re . compile ( r'.*%s-(\d+)\.zip$' % ( self . GetASANBaseName ( ) ) ) all_prefixes = document . findall ( namespace + 'Contents/' + namespace + 'Key' ) for prefix in all_prefixes : m = asan_regex . match ( prefix . text ) if m : try : revisions . append ( int ( m . group ( 1 ) ) ) except ValueError : pass else : all_prefixes = document . findall ( namespace + 'CommonPrefixes/' + namespace + 'Prefix' ) for prefix in all_prefixes : revnum = prefix . text [ prefix_len : - 1 ] try : revnum = int ( revnum ) revisions . append ( revnum ) except ValueError : pass return ( revisions , next_marker , githash_svn_dict ) if last_known_rev : revisions = [ ] next_marker = _GetMarkerForRev ( last_known_rev ) last_change_rev = GetChromiumRevision ( self , self . GetLastChangeURL ( ) ) if last_known_rev == last_change_rev : return [ ] else : ( revisions , next_marker , new_dict ) = _FetchAndParse ( self . GetListingURL ( ) ) self . githash_svn_dict . update ( new_dict ) last_change_rev = None while next_marker : sys . stdout . write ( '\rFetching revisions at marker %s' % next_marker ) sys . stdout . flush ( ) next_url = self . GetListingURL ( next_marker ) ( new_revisions , next_marker , new_dict ) = _FetchAndParse ( next_url ) revisions . extend ( new_revisions ) self . githash_svn_dict . update ( new_dict ) if last_change_rev and last_change_rev in new_revisions : break sys . stdout . write ( '\r' ) sys . stdout . flush ( ) return revisions
Parses the Google Storage directory listing into a list of revision
numbers.
Parses the Google Storage directory listing into a list of revision numbers .
"""Parses the Google Storage directory listing into a list of revision
    numbers.""" """Fetches a URL and returns a 2-Tuple of ([revisions], next-marker). If
      next-marker is not None, then the listing is a partial listing and another
      fetch should be performed with next-marker being the marker= GET
      parameter.""" # All nodes in the tree are namespaced. Get the root's tag name to extract # the namespace. Etree does namespaces as |{namespace}tag|. # Find the prefix (_listing_platform_dir) and whether or not the list is # truncated. # Get a list of all the revisions. # Non ASAN builds are in a <revision> directory. The ASAN builds are # flat # The <Prefix> nodes have content of the form of # |_listing_platform_dir/revision/|. Strip off the platform dir and the # trailing slash to just have a number. # Notes: # Ignore hash in chromium-browser-snapshots as they are invalid # Resulting in 404 error in fetching pages: # https://chromium.googlesource.com/chromium/src/+/[rev_hash] # Fetch the first list of revisions. # Optimization: Start paging at the last known revision (local cache). # Optimization: Stop paging at the last known revision (remote). # If the result list was truncated, refetch with the next marker. Do this # until an entire directory listing is done.
self
last_known_rev