gcs_util.py
def _native_download_file(meta, full_dst_file_name, max_concurrency):
        """
        Downloads the remote object to local file
        :param meta: file meta object (contains credentials and remote location)
        :param full_dst_file_name: path of the local file to download to
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        """
        logger = getLogger(__name__)

        if meta.get(u'presigned_url', None):
            # Use presigned url to download the object
            try:
                response = requests.get(meta[u'presigned_url'], stream=True)
                response.raise_for_status()

                with open(full_dst_file_name, 'wb') as fd:
                    for chunk in response.raw.stream(CONTENT_CHUNK_SIZE,
                                                     decode_content=False):
                        fd.write(chunk)
            except requests.exceptions.HTTPError as errh:
                logger.debug("GCS file download Http Error: %s", errh)
                # Presigned urls can be generated for any xml-api operation
                # offered by GCS. Hence the error codes expected are similar
                # to xml api.
                # https://cloud.google.com/storage/docs/xml-api/reference-status

                # According to the above resource, GCS recommends retrying
                # for the following error codes.
                if errh.response.status_code in [403, 408, 429, 500, 503]:
                    meta[u'last_error'] = errh
                    meta[u'result_status'] = ResultStatus.NEED_RETRY
                    return
                elif (errh.response.status_code == 400 and
                      (u'last_error' not in meta or meta[u'last_error'].errh.response.status_code != 400)):
                    # Only attempt to renew urls if this isn't the second time this happens
                    meta[u'last_error'] = errh
                    meta[u'result_status'] = ResultStatus.RENEW_PRESIGNED_URL
                    return
                # raise anything else
                raise errh
            except requests.exceptions.Timeout as errt:
                logger.debug("GCS file download Timeout Error: %s", errt)
                meta[u'last_error'] = errt
                meta[u'result_status'] = ResultStatus.NEED_RETRY
                return

            encryption_metadata = None

            if response.headers.get(GCS_METADATA_ENCRYPTIONDATAPROP, None):
                encryptiondata = json.loads(
                    response.headers[GCS_METADATA_ENCRYPTIONDATAPROP])

                if encryptiondata:
                    encryption_metadata = EncryptionMetadata(
                        key=encryptiondata[u'WrappedContentKey'][u'EncryptedKey'],
                        iv=encryptiondata[u'ContentEncryptionIV'],
                        matdesc=response.headers[GCS_METADATA_MATDESC_KEY]
                        if GCS_METADATA_MATDESC_KEY in response.headers
                        else None,
                    )

            # Sadly, we can only determine the src file size after we've
            # downloaded it, unlike the other cloud providers where the
            # metadata can be read beforehand.
            meta[u'src_file_size'] = os.path.getsize(full_dst_file_name)

            if meta[u'get_callback']:
                meta[u'get_callback'](
                    meta[u'src_file_name'],
                    meta[u'src_file_size'],
                    output_stream=meta[u'get_callback_output_stream'],
                    show_progress_bar=meta[u'show_progress_bar'])(
                    meta[u'src_file_size'])

            logger.debug(u'DONE getting a file')
            meta[u'result_status'] = ResultStatus.DOWNLOADED

            meta[GCS_FILE_HEADER_DIGEST] = response.headers.get(
                GCS_METADATA_SFC_DIGEST, None)
            meta[GCS_FILE_HEADER_CONTENT_LENGTH] = len(response.content)
            meta[GCS_FILE_HEADER_ENCRYPTION_METADATA] = encryption_metadata
        else:
            # Use the storage client to download the object
            # Currently not supported, given that we can't obtain
            # location-scoped access tokens
            logger.error(u"GCS download operation with an access token is "
                         u"currently unsupported")
            meta[u'result_status'] = ResultStatus.ERROR

        Downloads the remote object to local file
        :param meta: file meta object (contains credentials and remote location)
        :param full_dst_file_name: path of the local file to download to
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        
Downloads the remote object to local file
Downloads the remote object to local file
def _native_download_file(meta, full_dst_file_name, max_concurrency):
        logger = getLogger(__name__)
        if meta.get(u'presigned_url', None):
            try:
                response = requests.get(meta[u'presigned_url'], stream=True)
                response.raise_for_status()
                with open(full_dst_file_name, 'wb') as fd:
                    for chunk in response.raw.stream(CONTENT_CHUNK_SIZE,
                                                     decode_content=False):
                        fd.write(chunk)
            except requests.exceptions.HTTPError as errh:
                logger.debug("GCS file download Http Error: %s", errh)
                if errh.response.status_code in [403, 408, 429, 500, 503]:
                    meta[u'last_error'] = errh
                    meta[u'result_status'] = ResultStatus.NEED_RETRY
                    return
                elif (errh.response.status_code == 400 and
                      (u'last_error' not in meta or meta[u'last_error'].errh.response.status_code != 400)):
                    meta[u'last_error'] = errh
                    meta[u'result_status'] = ResultStatus.RENEW_PRESIGNED_URL
                    return
                raise errh
            except requests.exceptions.Timeout as errt:
                logger.debug("GCS file download Timeout Error: %s", errt)
                meta[u'last_error'] = errt
                meta[u'result_status'] = ResultStatus.NEED_RETRY
                return
            encryption_metadata = None
            if response.headers.get(GCS_METADATA_ENCRYPTIONDATAPROP, None):
                encryptiondata = json.loads(
                    response.headers[GCS_METADATA_ENCRYPTIONDATAPROP])
                if encryptiondata:
                    encryption_metadata = EncryptionMetadata(
                        key=encryptiondata[u'WrappedContentKey'][u'EncryptedKey'],
                        iv=encryptiondata[u'ContentEncryptionIV'],
                        matdesc=response.headers[GCS_METADATA_MATDESC_KEY]
                        if GCS_METADATA_MATDESC_KEY in response.headers
                        else None,
                    )
            meta[u'src_file_size'] = os.path.getsize(full_dst_file_name)
            if meta[u'get_callback']:
                meta[u'get_callback'](
                    meta[u'src_file_name'],
                    meta[u'src_file_size'],
                    output_stream=meta[u'get_callback_output_stream'],
                    show_progress_bar=meta[u'show_progress_bar'])(
                    meta[u'src_file_size'])
            logger.debug(u'DONE getting a file')
            meta[u'result_status'] = ResultStatus.DOWNLOADED
            meta[GCS_FILE_HEADER_DIGEST] = response.headers.get(
                GCS_METADATA_SFC_DIGEST, None)
            meta[GCS_FILE_HEADER_CONTENT_LENGTH] = len(response.content)
            meta[GCS_FILE_HEADER_ENCRYPTION_METADATA] = encryption_metadata
        else:
            logger.error(u"GCS download operation with an access token is "
                         u"currently unsupported")
            meta[u'result_status'] = ResultStatus.ERROR
def _native_download_file ( meta , full_dst_file_name , max_concurrency ) : logger = getLogger ( __name__ ) if meta . get ( u'presigned_url' , None ) : try : response = requests . get ( meta [ u'presigned_url' ] , stream = True ) response . raise_for_status ( ) with open ( full_dst_file_name , 'wb' ) as fd : for chunk in response . raw . stream ( CONTENT_CHUNK_SIZE , decode_content = False ) : fd . write ( chunk ) except requests . exceptions . HTTPError as errh : logger . debug ( "GCS file download Http Error: %s" , errh ) if errh . response . status_code in [ 403 , 408 , 429 , 500 , 503 ] : meta [ u'last_error' ] = errh meta [ u'result_status' ] = ResultStatus . NEED_RETRY return elif ( errh . response . status_code == 400 and ( u'last_error' not in meta or meta [ u'last_error' ] . errh . response . status_code != 400 ) ) : meta [ u'last_error' ] = errh meta [ u'result_status' ] = ResultStatus . RENEW_PRESIGNED_URL return raise errh except requests . exceptions . Timeout as errt : logger . debug ( "GCS file download Timeout Error: %s" , errt ) meta [ u'last_error' ] = errt meta [ u'result_status' ] = ResultStatus . NEED_RETRY return encryption_metadata = None if response . headers . get ( GCS_METADATA_ENCRYPTIONDATAPROP , None ) : encryptiondata = json . loads ( response . headers [ GCS_METADATA_ENCRYPTIONDATAPROP ] ) if encryptiondata : encryption_metadata = EncryptionMetadata ( key = encryptiondata [ u'WrappedContentKey' ] [ u'EncryptedKey' ] , iv = encryptiondata [ u'ContentEncryptionIV' ] , matdesc = response . headers [ GCS_METADATA_MATDESC_KEY ] if GCS_METADATA_MATDESC_KEY in response . headers else None , ) meta [ u'src_file_size' ] = os . path . getsize ( full_dst_file_name ) if meta [ u'get_callback' ] : meta [ u'get_callback' ] ( meta [ u'src_file_name' ] , meta [ u'src_file_size' ] , output_stream = meta [ u'get_callback_output_stream' ] , show_progress_bar = meta [ u'show_progress_bar' ] ) ( meta [ u'src_file_size' ] ) logger . debug ( u'DONE getting a file' ) meta [ u'result_status' ] = ResultStatus . DOWNLOADED meta [ GCS_FILE_HEADER_DIGEST ] = response . headers . get ( GCS_METADATA_SFC_DIGEST , None ) meta [ GCS_FILE_HEADER_CONTENT_LENGTH ] = len ( response . content ) meta [ GCS_FILE_HEADER_ENCRYPTION_METADATA ] = encryption_metadata else : logger . error ( u"GCS download operation with an access token is " u"currently unsupported" ) meta [ u'result_status' ] = ResultStatus . ERROR
Downloads the remote object to local file
Downloads the remote object to local file
"""
        Downloads the remote object to local file
        :param meta: file meta object (contains credentials and remote location)
        :param full_dst_file_name: path of the local file to download to
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        """ # Use presigned url to download the object # Presigned urls can be generated for any xml-api operation # offered by GCS. Hence the error codes expected are similar # to xml api. # https://cloud.google.com/storage/docs/xml-api/reference-status # According to the above resource, GCS recommends retrying # for the following error codes. # Only attempt to renew urls if this isn't the second time this happens # raise anything else # Sadly, we can only determine the src file size after we've # downloaded it, unlike the other cloud providers where the # metadata can be read beforehand. # Use the storage client to download the object # Currently not supported, given that we can't obtain # location-scoped access tokens
None, if successful. Otherwise, throws Exception.
None if successful . Otherwise throws Exception .
meta
file meta object (contains credentials and remote location)
file meta object ( contains credentials and remote location )
full_dst_file_name
path of the local file to download to
path of the local file to download to
max_concurrency
(inapplicable to GCS)
( inapplicable to GCS )