text: """Routines related to PyPI, indexes"""
  
  import sys
  import os
  import re
  import mimetypes
  import posixpath
  import pkg_resources
  
  from pip.log import logger
  from pip.util import Inf, normalize_name, splitext, is_prerelease
  from pip.exceptions import DistributionNotFound, BestVersionAlreadyInstalled,\
      InstallationError
  from pip.backwardcompat import urlparse, url2pathname
  from pip.download import PipSession, path_to_url2, url_to_path
  from pip.wheel import Wheel, wheel_ext, wheel_setuptools_support
  from pip.pep425tags import supported_tags, supported_tags_noarch, get_platform
  from pip._vendor import html5lib, requests
  from pip._vendor.requests.exceptions import SSLError
  
  
  __all__ = ['PackageFinder']
  
  
  DEFAULT_MIRROR_HOSTNAME = "last.pypi.python.org"
  
  INSECURE_SCHEMES = {
      "http": ["https"],
  }
  
  
  class PackageFinder(object):
      """This finds packages.
  
      This is meant to match easy_install's technique for looking for
      packages, by reading pages and looking for appropriate links
      """
  
      def __init__(self, find_links, index_urls,
              use_wheel=False, allow_external=[], allow_unverified=[],
              allow_all_external=False, allow_all_prereleases=False,
              process_dependency_links=False, session=None):
          self.find_links = find_links
          self.index_urls = index_urls
          self.dependency_links = []
          self.cache = PageCache()
          # These are boring links that have already been logged somehow:
          self.logged_links = set()
  
          self.use_wheel = use_wheel
  
          # Do we allow (safe and verifiable) externally hosted files?
          self.allow_external = set(normalize_name(n) for n in allow_external)
  
          # Which names are allowed to install insecure and unverifiable files?
          self.allow_unverified = set(
              normalize_name(n) for n in allow_unverified
          )
  
          # Do we allow all (safe and verifiable) externally hosted files?
          self.allow_all_external = allow_all_external
  
          # Stores if we ignored any external links so that we can instruct
          #   end users how to install them if no distributions are available
          self.need_warn_external = False
  
          # Stores if we ignored any unsafe links so that we can instruct
          #   end users how to install them if no distributions are available
          self.need_warn_unverified = False
  
          # Do we want to allow _all_ pre-releases?
          self.allow_all_prereleases = allow_all_prereleases
  
          # Do we process dependency links?
          self.process_dependency_links = process_dependency_links
  
          # The Session we'll use to make requests
          self.session = session or PipSession()
  
      @property
      def use_wheel(self):
          return self._use_wheel
  
      @use_wheel.setter
      def use_wheel(self, value):
          self._use_wheel = value
          if self._use_wheel and not wheel_setuptools_support():
              raise InstallationError("pip's wheel support requires setuptools >= 0.8 for dist-info support.")
  
      def add_dependency_links(self, links):
          ## FIXME: this shouldn't be global list this, it should only
          ## apply to requirements of the package that specifies the
          ## dependency_links value
          ## FIXME: also, we should track comes_from (i.e., use Link)
          if self.process_dependency_links:
              self.dependency_links.extend(links)
  
      def _sort_locations(self, locations):
          """
          Sort locations into "files" (archives) and "urls", and return
          a pair of lists (files,urls)
          """
          files = []
          urls = []
  
          # puts the url for the given file path into the appropriate list
          def sort_path(path):
              url = path_to_url2(path)
              if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
                  urls.append(url)
              else:
                  files.append(url)
  
          for url in locations:
  
              is_local_path = os.path.exists(url)
              is_file_url = url.startswith('file:')
              is_find_link = url in self.find_links
  
              if is_local_path or is_file_url:
                  if is_local_path:
                      path = url
                  else:
                      path = url_to_path(url)
                  if is_find_link and os.path.isdir(path):
                      path = os.path.realpath(path)
                      for item in os.listdir(path):
                          sort_path(os.path.join(path, item))
                  elif is_file_url and os.path.isdir(path):
                      urls.append(url)
                  elif os.path.isfile(path):
                      sort_path(path)
              else:
                  urls.append(url)
  
          return files, urls
  
      def _link_sort_key(self, link_tuple):
          """
          Function used to generate link sort key for link tuples.
          The greater the return value, the more preferred it is.
          If not finding wheels, then sorted by version only.
          If finding wheels, then the sort order is by version, then:
            1. existing installs
            2. wheels ordered via Wheel.support_index_min()
            3. source archives
          Note: it was considered to embed this logic into the Link
                comparison operators, but then different sdist links
                with the same version, would have to be considered equal
          """
          parsed_version, link, _ = link_tuple
          if self.use_wheel:
              support_num = len(supported_tags)
              if link == InfLink: # existing install
                  pri = 1
              elif link.wheel:
                  support_index = link.wheel.support_index_min()
                  if support_index is None:
                      raise InstallationError("%s is not a supported wheel for this platform. It can't be sorted." % link.wheel.filename)
                  pri = -(support_index)
              else: # sdist
                  pri = -(support_num)
              return (parsed_version, pri)
          else:
              return parsed_version
  
      def _sort_versions(self, applicable_versions):
          """
          Bring the latest version (and wheels) to the front, but maintain the existing ordering as secondary.
          See the docstring for `_link_sort_key` for details.
          This function is isolated for easier unit testing.
          """
          return sorted(applicable_versions, key=self._link_sort_key, reverse=True)
  
      def find_requirement(self, req, upgrade):
  
          def mkurl_pypi_url(url):
              loc = posixpath.join(url, url_name)
              # For maximum compatibility with easy_install, ensure the path
              # ends in a trailing slash.  Although this isn't in the spec
              # (and PyPI can handle it without the slash) some other index
              # implementations might break if they relied on easy_install's behavior.
              if not loc.endswith('/'):
                  loc = loc + '/'
              return loc
  
          url_name = req.url_name
          # Only check main index if index URL is given:
          main_index_url = None
          if self.index_urls:
              # Check that we have the url_name correctly spelled:
              main_index_url = Link(mkurl_pypi_url(self.index_urls[0]), trusted=True)
              # This will also cache the page, so it's okay that we get it again later:
              page = self._get_page(main_index_url, req)
              if page is None:
                  url_name = self._find_url_name(Link(self.index_urls[0], trusted=True), url_name, req) or req.url_name
  
          if url_name is not None:
              locations = [
                  mkurl_pypi_url(url)
                  for url in self.index_urls] + self.find_links
          else:
              locations = list(self.find_links)
          for version in req.absolute_versions:
              if url_name is not None and main_index_url is not None:
                  locations = [
                      posixpath.join(main_index_url.url, version)] + locations
  
          file_locations, url_locations = self._sort_locations(locations)
          _flocations, _ulocations = self._sort_locations(self.dependency_links)
          file_locations.extend(_flocations)
  
          # We trust every url that the user has given us whether it was given
          #   via --index-url or --find-links
          locations = [Link(url, trusted=True) for url in url_locations]
  
          # We explicitly do not trust links that came from dependency_links
          locations.extend([Link(url) for url in _ulocations])
  
          logger.debug('URLs to search for versions for %s:' % req)
          for location in locations:
              logger.debug('* %s' % location)
  
              # Determine if this url used a secure transport mechanism
              parsed = urlparse.urlparse(str(location))
              if parsed.scheme in INSECURE_SCHEMES:
                  secure_schemes = INSECURE_SCHEMES[parsed.scheme]
  
                  if len(secure_schemes) == 1:
                      ctx = (location, parsed.scheme, secure_schemes[0],
                             parsed.netloc)
                      logger.warn("%s uses an insecure transport scheme (%s). "
                                  "Consider using %s if %s has it available" %
                                  ctx)
                  elif len(secure_schemes) > 1:
                      ctx = (location, parsed.scheme, ", ".join(secure_schemes),
                                                                  parsed.netloc)
                      logger.warn("%s uses an insecure transport scheme (%s). "
                                  "Consider using one of %s if %s has any of "
                                  "them available" % ctx)
                  else:
                      ctx = (location, parsed.scheme)
                      logger.warn("%s uses an insecure transport scheme (%s)." %
                                  ctx)
  
          found_versions = []
          found_versions.extend(
              self._package_versions(
                  # We trust every directly linked archive in find_links
                  [Link(url, '-f', trusted=True) for url in self.find_links], req.name.lower()))
          page_versions = []
          for page in self._get_pages(locations, req):
              logger.debug('Analyzing links from page %s' % page.url)
              logger.indent += 2
              try:
                  page_versions.extend(self._package_versions(page.links, req.name.lower()))
              finally:
                  logger.indent -= 2
          dependency_versions = list(self._package_versions(
              [Link(url) for url in self.dependency_links], req.name.lower()))
          if dependency_versions:
              logger.info('dependency_links found: %s' % ', '.join([link.url for parsed, link, version in dependency_versions]))
          file_versions = list(self._package_versions(
                  [Link(url) for url in file_locations], req.name.lower()))
          if not found_versions and not page_versions and not dependency_versions and not file_versions:
              logger.fatal('Could not find any downloads that satisfy the requirement %s' % req)
  
              if self.need_warn_external:
                  logger.warn("Some externally hosted files were ignored (use "
                              "--allow-external %s to allow)." % req.name)
  
              if self.need_warn_unverified:
                  logger.warn("Some insecure and unverifiable files were ignored"
                              " (use --allow-unverified %s to allow)." %
                              req.name)
  
              raise DistributionNotFound('No distributions at all found for %s' % req)
          installed_version = []
          if req.satisfied_by is not None:
              installed_version = [(req.satisfied_by.parsed_version, InfLink, req.satisfied_by.version)]
          if file_versions:
              file_versions.sort(reverse=True)
              logger.info('Local files found: %s' % ', '.join([url_to_path(link.url) for parsed, link, version in file_versions]))
          #this is an intentional priority ordering
          all_versions = installed_version + file_versions + found_versions + page_versions + dependency_versions
          applicable_versions = []
          for (parsed_version, link, version) in all_versions:
              if version not in req.req:
                  logger.info("Ignoring link %s, version %s doesn't match %s"
                              % (link, version, ','.join([''.join(s) for s in req.req.specs])))
                  continue
              elif is_prerelease(version) and not (self.allow_all_prereleases or req.prereleases):
                  # If this version isn't the already installed one, then
                  #   ignore it if it's a pre-release.
                  if link is not InfLink:
                      logger.info("Ignoring link %s, version %s is a pre-release (use --pre to allow)." % (link, version))
                      continue
              applicable_versions.append((parsed_version, link, version))
          applicable_versions = self._sort_versions(applicable_versions)
          existing_applicable = bool([link for parsed_version, link, version in applicable_versions if link is InfLink])
          if not upgrade and existing_applicable:
              if applicable_versions[0][1] is InfLink:
                  logger.info('Existing installed version (%s) is most up-to-date and satisfies requirement'
                              % req.satisfied_by.version)
              else:
                  logger.info('Existing installed version (%s) satisfies requirement (most up-to-date version is %s)'
                              % (req.satisfied_by.version, applicable_versions[0][2]))
              return None
          if not applicable_versions:
              logger.fatal('Could not find a version that satisfies the requirement %s (from versions: %s)'
                           % (req, ', '.join([version for parsed_version, link, version in all_versions])))
  
              if self.need_warn_external:
                  logger.warn("Some externally hosted files were ignored (use "
                              "--allow-external to allow).")
  
              if self.need_warn_unverified:
                  logger.warn("Some insecure and unverifiable files were ignored"
                              " (use --allow-unverified %s to allow)." %
                              req.name)
  
              raise DistributionNotFound('No distributions matching the version for %s' % req)
          if applicable_versions[0][1] is InfLink:
              # We have an existing version, and its the best version
              logger.info('Installed version (%s) is most up-to-date (past versions: %s)'
                          % (req.satisfied_by.version, ', '.join([version for parsed_version, link, version in applicable_versions[1:]]) or 'none'))
              raise BestVersionAlreadyInstalled
          if len(applicable_versions) > 1:
              logger.info('Using version %s (newest of versions: %s)' %
                          (applicable_versions[0][2], ', '.join([version for parsed_version, link, version in applicable_versions])))
  
          selected_version = applicable_versions[0][1]
  
          if (selected_version.internal is not None
                  and not selected_version.internal):
              logger.warn("%s an externally hosted file and may be "
                          "unreliable" % req.name)
  
          if (selected_version.verifiable is not None
                  and not selected_version.verifiable):
              logger.warn("%s is potentially insecure and "
                          "unverifiable." % req.name)
  
          if selected_version._deprecated_regex:
              logger.deprecated(
                  "1.7",
                  "%s discovered using a deprecated method of parsing, "
                  "in the future it will no longer be discovered" % req.name
              )
  
          return selected_version
  
  
      def _find_url_name(self, index_url, url_name, req):
          """Finds the true URL name of a package, when the given name isn't quite correct.
          This is usually used to implement case-insensitivity."""
          if not index_url.url.endswith('/'):
              # Vaguely part of the PyPI API... weird but true.
              ## FIXME: bad to modify this?
              index_url.url += '/'
          page = self._get_page(index_url, req)
          if page is None:
              logger.fatal('Cannot fetch index base URL %s' % index_url)
              return
          norm_name = normalize_name(req.url_name)
          for link in page.links:
              base = posixpath.basename(link.path.rstrip('/'))
              if norm_name == normalize_name(base):
                  logger.notify('Real name of requirement %s is %s' % (url_name, base))
                  return base
          return None
  
      def _get_pages(self, locations, req):
          """
          Yields (page, page_url) from the given locations, skipping
          locations that have errors, and adding download/homepage links
          """
          all_locations = list(locations)
          seen = set()
  
          while all_locations:
              location = all_locations.pop(0)
              if location in seen:
                  continue
              seen.add(location)
  
              page = self._get_page(location, req)
              if page is None:
                  continue
  
              yield page
  
              for link in page.rel_links():
                  normalized = normalize_name(req.name).lower()
  
                  if (not normalized in self.allow_external
                          and not self.allow_all_external):
                      self.need_warn_external = True
                      logger.debug("Not searching %s for files because external "
                                   "urls are disallowed." % link)
                      continue
  
                  if (link.trusted is not None
                          and not link.trusted
                          and not normalized in self.allow_unverified):
                      logger.debug("Not searching %s for urls, it is an "
                                  "untrusted link and cannot produce safe or "
                                  "verifiable files." % link)
                      self.need_warn_unverified = True
                      continue
  
                  all_locations.append(link)
  
      _egg_fragment_re = re.compile(r'#egg=([^&]*)')
      _egg_info_re = re.compile(r'([a-z0-9_.]+)-([a-z0-9_.-]+)', re.I)
      _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$')
  
      def _sort_links(self, links):
          "Returns elements of links in order, non-egg links first, egg links second, while eliminating duplicates"
          eggs, no_eggs = [], []
          seen = set()
          for link in links:
              if link not in seen:
                  seen.add(link)
                  if link.egg_fragment:
                      eggs.append(link)
                  else:
                      no_eggs.append(link)
          return no_eggs + eggs
  
      def _package_versions(self, links, search_name):
          for link in self._sort_links(links):
              for v in self._link_package_versions(link, search_name):
                  yield v
  
      def _known_extensions(self):
          extensions = ('.tar.gz', '.tar.bz2', '.tar', '.tgz', '.zip')
          if self.use_wheel:
              return extensions + (wheel_ext,)
          return extensions
  
      def _link_package_versions(self, link, search_name):
          """
          Return an iterable of triples (pkg_resources_version_key,
          link, python_version) that can be extracted from the given
          link.
  
          Meant to be overridden by subclasses, not called by clients.
          """
          platform = get_platform()
  
          version = None
          if link.egg_fragment:
              egg_info = link.egg_fragment
          else:
              egg_info, ext = link.splitext()
              if not ext:
                  if link not in self.logged_links:
                      logger.debug('Skipping link %s; not a file' % link)
                      self.logged_links.add(link)
                  return []
              if egg_info.endswith('.tar'):
                  # Special double-extension case:
                  egg_info = egg_info[:-4]
                  ext = '.tar' + ext
              if ext not in self._known_extensions():
                  if link not in self.logged_links:
                      logger.debug('Skipping link %s; unknown archive format: %s' % (link, ext))
                      self.logged_links.add(link)
                  return []
              if "macosx10" in link.path and ext == '.zip':
                  if link not in self.logged_links:
                      logger.debug('Skipping link %s; macosx10 one' % (link))
                      self.logged_links.add(link)
                  return []
              if link.wheel:
                  if link.wheel.name.lower() != search_name.lower():
                      logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
                      return []
                  version = link.wheel.version
                  if not link.wheel.supported():
                      logger.debug('Skipping %s because it is not compatible with this Python' % link)
                      return []
  
                  # This is a dirty hack to prevent installing Binary Wheels from
                  #   PyPI unless it is a Windows Binary Wheel. This is paired
                  #   with a change to PyPI disabling uploads for the same. Once
                  #   we have a mechanism for enabling support for binary wheels
                  #   on linux that deals with the inherent problems of binary
                  #   distribution this can be removed.
                  comes_from = getattr(link, "comes_from", None)
                  if (not platform.startswith('win')
                      and comes_from is not None
                      and urlparse.urlparse(comes_from.url).netloc.endswith(
                                                          "pypi.python.org")):
                      if not link.wheel.supported(tags=supported_tags_noarch):
                          logger.debug(
                              "Skipping %s because it is a pypi-hosted binary "
                              "Wheel on an unsupported platform" % link
                          )
                          return []
  
          if not version:
              version = self._egg_info_matches(egg_info, search_name, link)
          if version is None:
              logger.debug('Skipping link %s; wrong project name (not %s)' % (link, search_name))
              return []
  
          if (link.internal is not None
                  and not link.internal
                  and not normalize_name(search_name).lower() in self.allow_external
                  and not self.allow_all_external):
              # We have a link that we are sure is external, so we should skip
              #   it unless we are allowing externals
              logger.debug("Skipping %s because it is externally hosted." % link)
              self.need_warn_external = True
              return []
  
          if (link.verifiable is not None
                  and not link.verifiable
                  and not (normalize_name(search_name).lower()
                      in self.allow_unverified)):
              # We have a link that we are sure we cannot verify it's integrity,
              #   so we should skip it unless we are allowing unsafe installs
              #   for this requirement.
              logger.debug("Skipping %s because it is an insecure and "
                           "unverifiable file." % link)
              self.need_warn_unverified = True
              return []
  
          match = self._py_version_re.search(version)
          if match:
              version = version[:match.start()]
              py_version = match.group(1)
              if py_version != sys.version[:3]:
                  logger.debug('Skipping %s because Python version is incorrect' % link)
                  return []
          logger.debug('Found link %s, version: %s' % (link, version))
          return [(pkg_resources.parse_version(version),
                 link,
                 version)]
  
      def _egg_info_matches(self, egg_info, search_name, link):
          match = self._egg_info_re.search(egg_info)
          if not match:
              logger.debug('Could not parse version from link: %s' % link)
              return None
          name = match.group(0).lower()
          # To match the "safe" name that pkg_resources creates:
          name = name.replace('_', '-')
          # project name and version must be separated by a dash
          look_for = search_name.lower() + "-"
          if name.startswith(look_for):
              return match.group(0)[len(look_for):]
          else:
              return None
  
      def _get_page(self, link, req):
          return HTMLPage.get_page(link, req,
              cache=self.cache,
              session=self.session,
          )
  
  
  class PageCache(object):
      """Cache of HTML pages"""
  
      failure_limit = 3
  
      def __init__(self):
          self._failures = {}
          self._pages = {}
          self._archives = {}
  
      def too_many_failures(self, url):
          return self._failures.get(url, 0) >= self.failure_limit
  
      def get_page(self, url):
          return self._pages.get(url)
  
      def is_archive(self, url):
          return self._archives.get(url, False)
  
      def set_is_archive(self, url, value=True):
          self._archives[url] = value
  
      def add_page_failure(self, url, level):
          self._failures[url] = self._failures.get(url, 0)+level
  
      def add_page(self, urls, page):
          for url in urls:
              self._pages[url] = page
  
  
  class HTMLPage(object):
      """Represents one page, along with its URL"""
  
      ## FIXME: these regexes are horrible hacks:
      _homepage_re = re.compile(r'<th>\s*home\s*page', re.I)
      _download_re = re.compile(r'<th>\s*download\s+url', re.I)
      _href_re = re.compile('href=(?:"([^"]*)"|\'([^\']*)\'|([^>\\s\\n]*))', re.I|re.S)
  
      def __init__(self, content, url, headers=None, trusted=None):
          self.content = content
          self.parsed = html5lib.parse(self.content, namespaceHTMLElements=False)
          self.url = url
          self.headers = headers
          self.trusted = trusted
  
      def __str__(self):
          return self.url
  
      @classmethod
      def get_page(cls, link, req, cache=None, skip_archives=True, session=None):
          if session is None:
              session = PipSession()
  
          url = link.url
          url = url.split('#', 1)[0]
          if cache.too_many_failures(url):
              return None
  
          # Check for VCS schemes that do not support lookup as web pages.
          from pip.vcs import VcsSupport
          for scheme in VcsSupport.schemes:
              if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
                  logger.debug('Cannot look at %(scheme)s URL %(link)s' % locals())
                  return None
  
          if cache is not None:
              inst = cache.get_page(url)
              if inst is not None:
                  return inst
          try:
              if skip_archives:
                  if cache is not None:
                      if cache.is_archive(url):
                          return None
                  filename = link.filename
                  for bad_ext in ['.tar', '.tar.gz', '.tar.bz2', '.tgz', '.zip']:
                      if filename.endswith(bad_ext):
                          content_type = cls._get_content_type(url,
                              session=session,
                          )
                          if content_type.lower().startswith('text/html'):
                              break
                          else:
                              logger.debug('Skipping page %s because of Content-Type: %s' % (link, content_type))
                              if cache is not None:
                                  cache.set_is_archive(url)
                              return None
              logger.debug('Getting page %s' % url)
  
              # Tack index.html onto file:// URLs that point to directories
              (scheme, netloc, path, params, query, fragment) = urlparse.urlparse(url)
              if scheme == 'file' and os.path.isdir(url2pathname(path)):
                  # add trailing slash if not present so urljoin doesn't trim final segment
                  if not url.endswith('/'):
                      url += '/'
                  url = urlparse.urljoin(url, 'index.html')
                  logger.debug(' file: URL is directory, getting %s' % url)
  
              resp = session.get(url)
              resp.raise_for_status()
  
              # The check for archives above only works if the url ends with
              #   something that looks like an archive. However that is not a
              #   requirement. For instance http://sourceforge.net/projects/docutils/files/docutils/0.8.1/docutils-0.8.1.tar.gz/download
              #   redirects to http://superb-dca3.dl.sourceforge.net/project/docutils/docutils/0.8.1/docutils-0.8.1.tar.gz
              #   Unless we issue a HEAD request on every url we cannot know
              #   ahead of time for sure if something is HTML or not. However we
              #   can check after we've downloaded it.
              content_type = resp.headers.get('Content-Type', 'unknown')
              if not content_type.lower().startswith("text/html"):
                  logger.debug('Skipping page %s because of Content-Type: %s' %
                                              (link, content_type))
                  if cache is not None:
                      cache.set_is_archive(url)
                  return None
  
              inst = cls(resp.text, resp.url, resp.headers, trusted=link.trusted)
          except requests.HTTPError as exc:
              level = 2 if exc.response.status_code == 404 else 1
              cls._handle_fail(req, link, exc, url, cache=cache, level=level)
          except requests.Timeout:
              cls._handle_fail(req, link, "timed out", url, cache=cache)
          except SSLError as exc:
              reason = ("There was a problem confirming the ssl certificate: "
                        "%s" % exc)
              cls._handle_fail(req, link, reason, url,
                  cache=cache,
                  level=2,
                  meth=logger.notify,
              )
          else:
              if cache is not None:
                  cache.add_page([url, resp.url], inst)
              return inst
  
      @staticmethod
      def _handle_fail(req, link, reason, url, cache=None, level=1, meth=None):
          if meth is None:
              meth = logger.info
  
          meth("Could not fetch URL %s: %s", link, reason)
          meth("Will skip URL %s when looking for download links for %s" %
               (link.url, req))
  
          if cache is not None:
              cache.add_page_failure(url, level)
  
      @staticmethod
      def _get_content_type(url, session=None):
          """Get the Content-Type of the given url, using a HEAD request"""
          if session is None:
              session = PipSession()
  
          scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
          if not scheme in ('http', 'https', 'ftp', 'ftps'):
              ## FIXME: some warning or something?
              ## assertion error?
              return ''
  
          resp = session.head(url, allow_redirects=True)
          resp.raise_for_status()
  
          return resp.headers.get("Content-Type", "")
  
      @property
      def api_version(self):
          if not hasattr(self, "_api_version"):
              _api_version = None
  
              metas = [x for x in self.parsed.findall(".//meta")
                          if x.get("name", "").lower() == "api-version"]
              if metas:
                  try:
                      _api_version = int(metas[0].get("value", None))
                  except (TypeError, ValueError):
                      _api_version = None
              self._api_version = _api_version
          return self._api_version
  
      @property
      def base_url(self):
          if not hasattr(self, "_base_url"):
              base = self.parsed.find(".//base")
              if base is not None and base.get("href"):
                  self._base_url = base.get("href")
              else:
                  self._base_url = self.url
          return self._base_url
  
      @property
      def links(self):
          """Yields all links in the page"""
          for anchor in self.parsed.findall(".//a"):
              if anchor.get("href"):
                  href = anchor.get("href")
                  url = self.clean_link(urlparse.urljoin(self.base_url, href))
  
                  # Determine if this link is internal. If that distinction
                  #   doesn't make sense in this context, then we don't make
                  #   any distinction.
                  internal = None
                  if self.api_version and self.api_version >= 2:
                      # Only api_versions >= 2 have a distinction between
                      #   external and internal links
                      internal = bool(anchor.get("rel")
                                  and "internal" in anchor.get("rel").split())
  
                  yield Link(url, self, internal=internal)
  
      def rel_links(self):
          for url in self.explicit_rel_links():
              yield url
          for url in self.scraped_rel_links():
              yield url
  
      def explicit_rel_links(self, rels=('homepage', 'download')):
          """Yields all links with the given relations"""
          rels = set(rels)
  
          for anchor in self.parsed.findall(".//a"):
              if anchor.get("rel") and anchor.get("href"):
                  found_rels = set(anchor.get("rel").split())
                  # Determine the intersection between what rels were found and
                  #   what rels were being looked for
                  if found_rels & rels:
                      href = anchor.get("href")
                      url = self.clean_link(urlparse.urljoin(self.base_url, href))
                      yield Link(url, self, trusted=False)
  
      def scraped_rel_links(self):
          # Can we get rid of this horrible horrible method?
          for regex in (self._homepage_re, self._download_re):
              match = regex.search(self.content)
              if not match:
                  continue
              href_match = self._href_re.search(self.content, pos=match.end())
              if not href_match:
                  continue
              url = href_match.group(1) or href_match.group(2) or href_match.group(3)
              if not url:
                  continue
              url = self.clean_link(urlparse.urljoin(self.base_url, url))
              yield Link(url, self, trusted=False, _deprecated_regex=True)
  
      _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
  
      def clean_link(self, url):
          """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
          the link, it will be rewritten to %20 (while not over-quoting
          % or other characters)."""
          return self._clean_re.sub(
              lambda match: '%%%2x' % ord(match.group(0)), url)
  
  
  class Link(object):
  
      def __init__(self, url, comes_from=None, internal=None, trusted=None,
              _deprecated_regex=False):
          self.url = url
          self.comes_from = comes_from
          self.internal = internal
          self.trusted = trusted
          self._deprecated_regex = _deprecated_regex
  
          # Set whether it's a wheel
          self.wheel = None
          if url != Inf and self.splitext()[1] == wheel_ext:
              self.wheel = Wheel(self.filename)
  
      def __str__(self):
          if self.comes_from:
              return '%s (from %s)' % (self.url, self.comes_from)
          else:
              return str(self.url)
  
      def __repr__(self):
          return '<Link %s>' % self
  
      def __eq__(self, other):
          return self.url == other.url
  
      def __ne__(self, other):
          return self.url != other.url
  
      def __lt__(self, other):
          return self.url < other.url
  
      def __le__(self, other):
          return self.url <= other.url
  
      def __gt__(self, other):
          return self.url > other.url
  
      def __ge__(self, other):
          return self.url >= other.url
  
      def __hash__(self):
          return hash(self.url)
  
      @property
      def filename(self):
          _, netloc, path, _, _ = urlparse.urlsplit(self.url)
          name = posixpath.basename(path.rstrip('/')) or netloc
          assert name, ('URL %r produced no filename' % self.url)
          return name
  
      @property
      def scheme(self):
          return urlparse.urlsplit(self.url)[0]
  
      @property
      def path(self):
          return urlparse.urlsplit(self.url)[2]
  
      def splitext(self):
          return splitext(posixpath.basename(self.path.rstrip('/')))
  
      @property
      def url_without_fragment(self):
          scheme, netloc, path, query, fragment = urlparse.urlsplit(self.url)
          return urlparse.urlunsplit((scheme, netloc, path, query, None))
  
      _egg_fragment_re = re.compile(r'#egg=([^&]*)')
  
      @property
      def egg_fragment(self):
          match = self._egg_fragment_re.search(self.url)
          if not match:
              return None
          return match.group(1)
  
      _hash_re = re.compile(r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)')
  
      @property
      def hash(self):
          match = self._hash_re.search(self.url)
          if match:
              return match.group(2)
          return None
  
      @property
      def hash_name(self):
          match = self._hash_re.search(self.url)
          if match:
              return match.group(1)
          return None
  
      @property
      def show_url(self):
          return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0])
  
      @property
      def verifiable(self):
          """
          Returns True if this link can be verified after download, False if it
          cannot, and None if we cannot determine.
          """
          trusted = self.trusted or getattr(self.comes_from, "trusted", None)
          if trusted is not None and trusted:
              # This link came from a trusted source. It *may* be verifiable but
              #   first we need to see if this page is operating under the new
              #   API version.
              try:
                  api_version = getattr(self.comes_from, "api_version", None)
                  api_version = int(api_version)
              except (ValueError, TypeError):
                  api_version = None
  
              if api_version is None or api_version <= 1:
                  # This link is either trusted, or it came from a trusted,
                  #   however it is not operating under the API version 2 so
                  #   we can't make any claims about if it's safe or not
                  return
  
              if self.hash:
                  # This link came from a trusted source and it has a hash, so we
                  #   can consider it safe.
                  return True
              else:
                  # This link came from a trusted source, using the new API
                  #   version, and it does not have a hash. It is NOT verifiable
                  return False
          elif trusted is not None:
              # This link came from an untrusted source and we cannot trust it
              return False
  
  #An "Infinite Link" that compares greater than other links
  InfLink = Link(Inf) #this object is not currently used as a sortable
  
  
  def get_requirement_from_url(url):
      """Get a requirement from the URL, if possible.  This looks for #egg
      in the URL"""
      link = Link(url)
      egg_info = link.egg_fragment
      if not egg_info:
          egg_info = splitext(link.filename)[0]
      return package_to_requirement(egg_info)
  
  
  def package_to_requirement(package_name):
      """Translate a name like Foo-1.2 to Foo==1.3"""
      match = re.search(r'^(.*?)-(dev|\d.*)', package_name)
      if match:
          name = match.group(1)
          version = match.group(2)
      else:
          name = package_name
          version = ''
      if version:
          return '%s==%s' % (name, version)
      else:
          return name
  