text: # -*- coding: utf-8 -*-
  #
  # Copyright (C) 2013 Vinay Sajip.
  # Licensed to the Python Software Foundation under a contributor agreement.
  # See LICENSE.txt and CONTRIBUTORS.txt.
  #
  import hashlib
  import logging
  import os
  import shutil
  import subprocess
  import tempfile
  try:
      from threading import Thread
  except ImportError:
      from dummy_threading import Thread
  
  from . import DistlibException
  from .compat import (HTTPBasicAuthHandler, Request, HTTPPasswordMgr,
                       urlparse, build_opener, string_types)
  from .util import zip_dir, ServerProxy
  
  logger = logging.getLogger(__name__)
  
  DEFAULT_INDEX = 'https://pypi.org/pypi'
  DEFAULT_REALM = 'pypi'
  
  class PackageIndex(object):
      """
      This class represents a package index compatible with PyPI, the Python
      Package Index.
      """
  
      boundary = b'----------ThIs_Is_tHe_distlib_index_bouNdaRY_$'
  
      def __init__(self, url=None):
          """
          Initialise an instance.
  
          :param url: The URL of the index. If not specified, the URL for PyPI is
                      used.
          """
          self.url = url or DEFAULT_INDEX
          self.read_configuration()
          scheme, netloc, path, params, query, frag = urlparse(self.url)
          if params or query or frag or scheme not in ('http', 'https'):
              raise DistlibException('invalid repository: %s' % self.url)
          self.password_handler = None
          self.ssl_verifier = None
          self.gpg = None
          self.gpg_home = None
          with open(os.devnull, 'w') as sink:
              # Use gpg by default rather than gpg2, as gpg2 insists on
              # prompting for passwords
              for s in ('gpg', 'gpg2'):
                  try:
                      rc = subprocess.check_call([s, '--version'], stdout=sink,
                                                 stderr=sink)
                      if rc == 0:
                          self.gpg = s
                          break
                  except OSError:
                      pass
  
      def _get_pypirc_command(self):
          """
          Get the distutils command for interacting with PyPI configurations.
          :return: the command.
          """
          from .util import _get_pypirc_command as cmd
          return cmd()
  
      def read_configuration(self):
          """
          Read the PyPI access configuration as supported by distutils. This populates
          ``username``, ``password``, ``realm`` and ``url`` attributes from the
          configuration.
          """
          from .util import _load_pypirc
          cfg = _load_pypirc(self)
          self.username = get('username')
          self.password = get('password')
          self.realm = get('realm', 'pypi')
          self.url = get('repository', self.url)
  
      def save_configuration(self):
          """
          Save the PyPI access configuration. You must have set ``username`` and
          ``password`` attributes before calling this method.
          """
          self.check_credentials()
          from .util import _store_pypirc
          _store_pypirc(self)
  
      def check_credentials(self):
          """
          Check that ``username`` and ``password`` have been set, and raise an
          exception if not.
          """
          if self.username is None or self.password is None:
              raise DistlibException('username and password must be set')
          pm = HTTPPasswordMgr()
          _, netloc, _, _, _, _ = urlparse(self.url)
          pm.add_password(self.realm, netloc, self.username, self.password)
          self.password_handler = HTTPBasicAuthHandler(pm)
  
      def register(self, metadata):
          """
          Register a distribution on PyPI, using the provided metadata.
  
          :param metadata: A :class:`Metadata` instance defining at least a name
                           and version number for the distribution to be
                           registered.
          :return: The HTTP response received from PyPI upon submission of the
                  request.
          """
          self.check_credentials()
          metadata.validate()
          d = metadata.todict()
          d[':action'] = 'verify'
          request = self.encode_request(d.items(), [])
          response = self.send_request(request)
          d[':action'] = 'submit'
          request = self.encode_request(d.items(), [])
          return self.send_request(request)
  
      def _reader(self, name, stream, outbuf):
          """
          Thread runner for reading lines of from a subprocess into a buffer.
  
          :param name: The logical name of the stream (used for logging only).
          :param stream: The stream to read from. This will typically a pipe
                         connected to the output stream of a subprocess.
          :param outbuf: The list to append the read lines to.
          """
          while True:
              s = stream.readline()
              if not s:
                  break
              s = s.decode('utf-8').rstrip()
              outbuf.append(s)
              logger.debug('%s: %s' % (name, s))
          stream.close()
  
      def get_sign_command(self, filename, signer, sign_password,
                           keystore=None):
          """
          Return a suitable command for signing a file.
  
          :param filename: The pathname to the file to be signed.
          :param signer: The identifier of the signer of the file.
          :param sign_password: The passphrase for the signer's
                                private key used for signing.
          :param keystore: The path to a directory which contains the keys
                           used in verification. If not specified, the
                           instance's ``gpg_home`` attribute is used instead.
          :return: The signing command as a list suitable to be
                   passed to :class:`subprocess.Popen`.
          """
          cmd = [self.gpg, '--status-fd', '2', '--no-tty']
          if keystore is None:
              keystore = self.gpg_home
          if keystore:
              cmd.extend(['--homedir', keystore])
          if sign_password is not None:
              cmd.extend(['--batch', '--passphrase-fd', '0'])
          td = tempfile.mkdtemp()
          sf = os.path.join(td, os.path.basename(filename) + '.asc')
          cmd.extend(['--detach-sign', '--armor', '--local-user',
                      signer, '--output', sf, filename])
          logger.debug('invoking: %s', ' '.join(cmd))
          return cmd, sf
  
      def run_command(self, cmd, input_data=None):
          """
          Run a command in a child process , passing it any input data specified.
  
          :param cmd: The command to run.
          :param input_data: If specified, this must be a byte string containing
                             data to be sent to the child process.
          :return: A tuple consisting of the subprocess' exit code, a list of
                   lines read from the subprocess' ``stdout``, and a list of
                   lines read from the subprocess' ``stderr``.
          """
          kwargs = {
              'stdout': subprocess.PIPE,
              'stderr': subprocess.PIPE,
          }
          if input_data is not None:
              kwargs['stdin'] = subprocess.PIPE
          stdout = []
          stderr = []
          p = subprocess.Popen(cmd, **kwargs)
          # We don't use communicate() here because we may need to
          # get clever with interacting with the command
          t1 = Thread(target=self._reader, args=('stdout', p.stdout, stdout))
          t1.start()
          t2 = Thread(target=self._reader, args=('stderr', p.stderr, stderr))
          t2.start()
          if input_data is not None:
              p.stdin.write(input_data)
              p.stdin.close()
  
          p.wait()
          t1.join()
          t2.join()
          return p.returncode, stdout, stderr
  
      def sign_file(self, filename, signer, sign_password, keystore=None):
          """
          Sign a file.
  
          :param filename: The pathname to the file to be signed.
          :param signer: The identifier of the signer of the file.
          :param sign_password: The passphrase for the signer's
                                private key used for signing.
          :param keystore: The path to a directory which contains the keys
                           used in signing. If not specified, the instance's
                           ``gpg_home`` attribute is used instead.
          :return: The absolute pathname of the file where the signature is
                   stored.
          """
          cmd, sig_file = self.get_sign_command(filename, signer, sign_password,
                                                keystore)
          rc, stdout, stderr = self.run_command(cmd,
                                                sign_password.encode('utf-8'))
          if rc != 0:
              raise DistlibException('sign command failed with error '
                                     'code %s' % rc)
          return sig_file
  
      def upload_file(self, metadata, filename, signer=None, sign_password=None,
                      filetype='sdist', pyversion='source', keystore=None):
          """
          Upload a release file to the index.
  
          :param metadata: A :class:`Metadata` instance defining at least a name
                           and version number for the file to be uploaded.
          :param filename: The pathname of the file to be uploaded.
          :param signer: The identifier of the signer of the file.
          :param sign_password: The passphrase for the signer's
                                private key used for signing.
          :param filetype: The type of the file being uploaded. This is the
                          distutils command which produced that file, e.g.
                          ``sdist`` or ``bdist_wheel``.
          :param pyversion: The version of Python which the release relates
                            to. For code compatible with any Python, this would
                            be ``source``, otherwise it would be e.g. ``3.2``.
          :param keystore: The path to a directory which contains the keys
                           used in signing. If not specified, the instance's
                           ``gpg_home`` attribute is used instead.
          :return: The HTTP response received from PyPI upon submission of the
                  request.
          """
          self.check_credentials()
          if not os.path.exists(filename):
              raise DistlibException('not found: %s' % filename)
          metadata.validate()
          d = metadata.todict()
          sig_file = None
          if signer:
              if not self.gpg:
                  logger.warning('no signing program available - not signed')
              else:
                  sig_file = self.sign_file(filename, signer, sign_password,
                                            keystore)
          with open(filename, 'rb') as f:
              file_data = f.read()
          md5_digest = hashlib.md5(file_data).hexdigest()
          sha256_digest = hashlib.sha256(file_data).hexdigest()
          d.update({
              ':action': 'file_upload',
              'protocol_version': '1',
              'filetype': filetype,
              'pyversion': pyversion,
              'md5_digest': md5_digest,
              'sha256_digest': sha256_digest,
          })
          files = [('content', os.path.basename(filename), file_data)]
          if sig_file:
              with open(sig_file, 'rb') as f:
                  sig_data = f.read()
              files.append(('gpg_signature', os.path.basename(sig_file),
                           sig_data))
              shutil.rmtree(os.path.dirname(sig_file))
          request = self.encode_request(d.items(), files)
          return self.send_request(request)
  
      def upload_documentation(self, metadata, doc_dir):
          """
          Upload documentation to the index.
  
          :param metadata: A :class:`Metadata` instance defining at least a name
                           and version number for the documentation to be
                           uploaded.
          :param doc_dir: The pathname of the directory which contains the
                          documentation. This should be the directory that
                          contains the ``index.html`` for the documentation.
          :return: The HTTP response received from PyPI upon submission of the
                  request.
          """
          self.check_credentials()
          if not os.path.isdir(doc_dir):
              raise DistlibException('not a directory: %r' % doc_dir)
          fn = os.path.join(doc_dir, 'index.html')
          if not os.path.exists(fn):
              raise DistlibException('not found: %r' % fn)
          metadata.validate()
          name, version = metadata.name, metadata.version
          zip_data = zip_dir(doc_dir).getvalue()
          fields = [(':action', 'doc_upload'),
                    ('name', name), ('version', version)]
          files = [('content', name, zip_data)]
          request = self.encode_request(fields, files)
          return self.send_request(request)
  
      def get_verify_command(self, signature_filename, data_filename,
                             keystore=None):
          """
          Return a suitable command for verifying a file.
  
          :param signature_filename: The pathname to the file containing the
                                     signature.
          :param data_filename: The pathname to the file containing the
                                signed data.
          :param keystore: The path to a directory which contains the keys
                           used in verification. If not specified, the
                           instance's ``gpg_home`` attribute is used instead.
          :return: The verifying command as a list suitable to be
                   passed to :class:`subprocess.Popen`.
          """
          cmd = [self.gpg, '--status-fd', '2', '--no-tty']
          if keystore is None:
              keystore = self.gpg_home
          if keystore:
              cmd.extend(['--homedir', keystore])
          cmd.extend(['--verify', signature_filename, data_filename])
          logger.debug('invoking: %s', ' '.join(cmd))
          return cmd
  
      def verify_signature(self, signature_filename, data_filename,
                           keystore=None):
          """
          Verify a signature for a file.
  
          :param signature_filename: The pathname to the file containing the
                                     signature.
          :param data_filename: The pathname to the file containing the
                                signed data.
          :param keystore: The path to a directory which contains the keys
                           used in verification. If not specified, the
                           instance's ``gpg_home`` attribute is used instead.
          :return: True if the signature was verified, else False.
          """
          if not self.gpg:
              raise DistlibException('verification unavailable because gpg '
                                     'unavailable')
          cmd = self.get_verify_command(signature_filename, data_filename,
                                        keystore)
          rc, stdout, stderr = self.run_command(cmd)
          if rc not in (0, 1):
              raise DistlibException('verify command failed with error '
                               'code %s' % rc)
          return rc == 0
  
      def download_file(self, url, destfile, digest=None, reporthook=None):
          """
          This is a convenience method for downloading a file from an URL.
          Normally, this will be a file from the index, though currently
          no check is made for this (i.e. a file can be downloaded from
          anywhere).
  
          The method is just like the :func:`urlretrieve` function in the
          standard library, except that it allows digest computation to be
          done during download and checking that the downloaded data
          matched any expected value.
  
          :param url: The URL of the file to be downloaded (assumed to be
                      available via an HTTP GET request).
          :param destfile: The pathname where the downloaded file is to be
                           saved.
          :param digest: If specified, this must be a (hasher, value)
                         tuple, where hasher is the algorithm used (e.g.
                         ``'md5'``) and ``value`` is the expected value.
          :param reporthook: The same as for :func:`urlretrieve` in the
                             standard library.
          """
          if digest is None:
              digester = None
              logger.debug('No digest specified')
          else:
              if isinstance(digest, (list, tuple)):
                  hasher, digest = digest
              else:
                  hasher = 'md5'
              digester = getattr(hashlib, hasher)()
              logger.debug('Digest specified: %s' % digest)
          # The following code is equivalent to urlretrieve.
          # We need to do it this way so that we can compute the
          # digest of the file as we go.
          with open(destfile, 'wb') as dfp:
              # addinfourl is not a context manager on 2.x
              # so we have to use try/finally
              sfp = self.send_request(Request(url))
              try:
                  headers = sfp.info()
                  blocksize = 8192
                  size = -1
                  read = 0
                  blocknum = 0
                  if "content-length" in headers:
                      size = int(headers["Content-Length"])
                  if reporthook:
                      reporthook(blocknum, blocksize, size)
                  while True:
                      block = sfp.read(blocksize)
                      if not block:
                          break
                      read += len(block)
                      dfp.write(block)
                      if digester:
                          digester.update(block)
                      blocknum += 1
                      if reporthook:
                          reporthook(blocknum, blocksize, size)
              finally:
                  sfp.close()
  
          # check that we got the whole file, if we can
          if size >= 0 and read < size:
              raise DistlibException(
                  'retrieval incomplete: got only %d out of %d bytes'
                  % (read, size))
          # if we have a digest, it must match.
          if digester:
              actual = digester.hexdigest()
              if digest != actual:
                  raise DistlibException('%s digest mismatch for %s: expected '
                                         '%s, got %s' % (hasher, destfile,
                                                         digest, actual))
              logger.debug('Digest verified: %s', digest)
  
      def send_request(self, req):
          """
          Send a standard library :class:`Request` to PyPI and return its
          response.
  
          :param req: The request to send.
          :return: The HTTP response from PyPI (a standard library HTTPResponse).
          """
          handlers = []
          if self.password_handler:
              handlers.append(self.password_handler)
          if self.ssl_verifier:
              handlers.append(self.ssl_verifier)
          opener = build_opener(*handlers)
          return opener.open(req)
  
      def encode_request(self, fields, files):
          """
          Encode fields and files for posting to an HTTP server.
  
          :param fields: The fields to send as a list of (fieldname, value)
                         tuples.
          :param files: The files to send as a list of (fieldname, filename,
                        file_bytes) tuple.
          """
          # Adapted from packaging, which in turn was adapted from
          # http://code.activestate.com/recipes/146306
  
          parts = []
          boundary = self.boundary
          for k, values in fields:
              if not isinstance(values, (list, tuple)):
                  values = [values]
  
              for v in values:
                  parts.extend((
                      b'--' + boundary,
                      ('Content-Disposition: form-data; name="%s"' %
                       k).encode('utf-8'),
                      b'',
                      v.encode('utf-8')))
          for key, filename, value in files:
              parts.extend((
                  b'--' + boundary,
                  ('Content-Disposition: form-data; name="%s"; filename="%s"' %
                   (key, filename)).encode('utf-8'),
                  b'',
                  value))
  
          parts.extend((b'--' + boundary + b'--', b''))
  
          body = b'\r\n'.join(parts)
          ct = b'multipart/form-data; boundary=' + boundary
          headers = {
              'Content-type': ct,
              'Content-length': str(len(body))
          }
          return Request(self.url, body, headers)
  
      def search(self, terms, operator=None):
          if isinstance(terms, string_types):
              terms = {'name': terms}
          rpc_proxy = ServerProxy(self.url, timeout=3.0)
          try:
              return rpc_proxy.search(terms, operator or 'and')
          finally:
              rpc_proxy('close')()
  