s3transfer/download.py
def _main(self, client, bucket, key, fileobj, extra_args, callbacks,
              max_attempts, download_output_manager, io_chunksize,
              start_index=0, bandwidth_limiter=None):
        """Downloads an object and places content into io queue

        :param client: The client to use when calling GetObject
        :param bucket: The bucket to download from
        :param key: The key to download from
        :param fileobj: The file handle to write content to
        :param exta_args: Any extra arguements to include in GetObject request
        :param callbacks: List of progress callbacks to invoke on download
        :param max_attempts: The number of retries to do when downloading
        :param download_output_manager: The download output manager associated
            with the current download.
        :param io_chunksize: The size of each io chunk to read from the
            download stream and queue in the io queue.
        :param start_index: The location in the file to start writing the
            content of the key to.
        :param bandwidth_limiter: The bandwidth limiter to use when throttling
            the downloading of data in streams.
        """
        last_exception = None
        for i in range(max_attempts):
            try:
                response = client.get_object(
                    Bucket=bucket, Key=key, **extra_args)
                streaming_body = StreamReaderProgress(
                    response['Body'], callbacks)
                if bandwidth_limiter:
                    streaming_body = \
                        bandwidth_limiter.get_bandwith_limited_stream(
                            streaming_body, self._transfer_coordinator)

                current_index = start_index
                chunks = DownloadChunkIterator(streaming_body, io_chunksize)
                for chunk in chunks:
                    # If the transfer is done because of a cancellation
                    # or error somewhere else, stop trying to submit more
                    # data to be written and break out of the download.
                    if not self._transfer_coordinator.done():
                        self._handle_io(
                            download_output_manager, fileobj, chunk,
                            current_index
                        )
                        current_index += len(chunk)
                    else:
                        return
                return
            except S3_RETRYABLE_DOWNLOAD_ERRORS as e:
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                # Also invoke the progress callbacks to indicate that we
                # are trying to download the stream again and all progress
                # for this GetObject has been lost.
                invoke_progress_callbacks(
                    callbacks, start_index - current_index)
                continue
        raise RetriesExceededError(last_exception)
def _main(self, client, bucket, key, fileobj, extra_args, callbacks,
              max_attempts, download_output_manager, io_chunksize,
              start_index=0, bandwidth_limiter=None):
        """Downloads an object and places content into io queue

        :param client: The client to use when calling GetObject
        :param bucket: The bucket to download from
        :param key: The key to download from
        :param fileobj: The file handle to write content to
        :param exta_args: Any extra arguements to include in GetObject request
        :param callbacks: List of progress callbacks to invoke on download
        :param max_attempts: The number of retries to do when downloading
        :param download_output_manager: The download output manager associated
            with the current download.
        :param io_chunksize: The size of each io chunk to read from the
            download stream and queue in the io queue.
        :param start_index: The location in the file to start writing the
            content of the key to.
        :param bandwidth_limiter: The bandwidth limiter to use when throttling
            the downloading of data in streams.
        """
        last_exception = None
        for i in range(max_attempts):
            try:
                response = client.get_object(
                    Bucket=bucket, Key=key, **extra_args)
                streaming_body = StreamReaderProgress(
                    response['Body'], callbacks)
                if bandwidth_limiter:
                    streaming_body = \
                        bandwidth_limiter.get_bandwith_limited_stream(
                            streaming_body, self._transfer_coordinator)

                current_index = start_index
                chunks = DownloadChunkIterator(streaming_body, io_chunksize)
                for chunk in chunks:
                    # If the transfer is done because of a cancellation
                    # or error somewhere else, stop trying to submit more
                    # data to be written and break out of the download.
                    if not self._transfer_coordinator.done():
                        self._handle_io(
                            download_output_manager, fileobj, chunk,
                            current_index
                        )
                        current_index += len(chunk)
                    else:
                        return
                return
            except S3_RETRYABLE_DOWNLOAD_ERRORS as e:
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                # Also invoke the progress callbacks to indicate that we
                # are trying to download the stream again and all progress
                # for this GetObject has been lost.
                invoke_progress_callbacks(
                    callbacks, start_index - current_index)
                continue
        raise RetriesExceededError(last_exception)
def _main ( self , client , bucket , key , fileobj , extra_args , callbacks , max_attempts , download_output_manager , io_chunksize , start_index = 0 , bandwidth_limiter = None ) : last_exception = None for i in range ( max_attempts ) : try : response = client . get_object ( Bucket = bucket , Key = key , * * extra_args ) streaming_body = StreamReaderProgress ( response [ 'Body' ] , callbacks ) if bandwidth_limiter : streaming_body = bandwidth_limiter . get_bandwith_limited_stream ( streaming_body , self . _transfer_coordinator ) current_index = start_index chunks = DownloadChunkIterator ( streaming_body , io_chunksize ) for chunk in chunks : # If the transfer is done because of a cancellation # or error somewhere else, stop trying to submit more # data to be written and break out of the download. if not self . _transfer_coordinator . done ( ) : self . _handle_io ( download_output_manager , fileobj , chunk , current_index ) current_index += len ( chunk ) else : return return except S3_RETRYABLE_DOWNLOAD_ERRORS as e : logger . debug ( "Retrying exception caught (%s), " "retrying request, (attempt %s / %s)" , e , i , max_attempts , exc_info = True ) last_exception = e # Also invoke the progress callbacks to indicate that we # are trying to download the stream again and all progress # for this GetObject has been lost. invoke_progress_callbacks ( callbacks , start_index - current_index ) continue raise RetriesExceededError ( last_exception )
Downloads an object and places content into io queue

        :param client: The client to use when calling GetObject
        :param bucket: The bucket to download from
        :param key: The key to download from
        :param fileobj: The file handle to write content to
        :param exta_args: Any extra arguements to include in GetObject request
        :param callbacks: List of progress callbacks to invoke on download
        :param max_attempts: The number of retries to do when downloading
        :param download_output_manager: The download output manager associated
            with the current download.
        :param io_chunksize: The size of each io chunk to read from the
            download stream and queue in the io queue.
        :param start_index: The location in the file to start writing the
            content of the key to.
        :param bandwidth_limiter: The bandwidth limiter to use when throttling
            the downloading of data in streams.
Downloads an object and places content into io queue