gcs_util.py
def upload_file(data_file, meta, encryption_metadata, max_concurrency):
        """
        Uploads the local file to remote storage
        :param data_file: file path on local system
        :param meta: file meta object (contains credentials and remote location)
        :param encryption_metadata: encryption metadata to be set on object
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        """
        logger = getLogger(__name__)

        if meta.get(u'presigned_url', None):
            # Use presigned url to upload the object

            content_encoding = ""
            if meta.get(u'dst_compression_type') is not None:
                content_encoding = meta[u'dst_compression_type'][u'name'].lower()

            # We set the contentEncoding to blank for GZIP files. We don't
            # want GCS to think our gzip files are gzips because it makes
            # them download uncompressed, and none of the other providers do
            # that. There's essentially no way for us to prevent that
            # behavior. Bad Google.
            if content_encoding and content_encoding == u'gzip':
                content_encoding = ""

            gcs_headers = {
                HTTP_HEADER_CONTENT_ENCODING: content_encoding,
                GCS_METADATA_SFC_DIGEST: meta[SHA256_DIGEST],
            }

            if encryption_metadata:
                gcs_headers.update({
                    GCS_METADATA_ENCRYPTIONDATAPROP: json.dumps({
                        u'EncryptionMode': u'FullBlob',
                        u'WrappedContentKey': {
                            u'KeyId': u'symmKey1',
                            u'EncryptedKey': encryption_metadata.key,
                            u'Algorithm': u'AES_CBC_256'
                        },
                        u'EncryptionAgent': {
                            u'Protocol': '1.0',
                            u'EncryptionAlgorithm': u'AES_CBC_256',
                        },
                        u'ContentEncryptionIV': encryption_metadata.iv,
                        u'KeyWrappingMetadata': {
                            u'EncryptionLibrary': u'Java 5.3.0'
                        }
                    }),
                    GCS_METADATA_MATDESC_KEY: encryption_metadata.matdesc
                })

            with open(data_file, 'rb') as fd:
                try:
                    response = requests.put(
                        meta[u'presigned_url'],
                        data=fd,
                        headers=gcs_headers)
                    response.raise_for_status()
                except requests.exceptions.HTTPError as errh:
                    logger.debug("GCS file upload Http Error: %s", errh)
                    # Presigned urls can be generated for any xml-api operation
                    # offered by GCS. Hence the error codes expected are similar
                    # to xml api.
                    # https://cloud.google.com/storage/docs/xml-api/reference-status

                    # According to the above resource, GCS recommends retrying
                    # for the following error codes.
                    if errh.response.status_code in [403, 408, 429, 500, 503]:
                        meta[u'last_error'] = errh
                        meta[u'result_status'] = ResultStatus.NEED_RETRY
                        return
                    elif (errh.response.status_code == 400 and
                          (u'last_error' not in meta or meta[u'last_error'].response.status_code != 400)):
                        # Only attempt to renew urls if this isn't the second time this happens
                        meta[u'last_error'] = errh
                        meta[u'result_status'] = ResultStatus.RENEW_PRESIGNED_URL
                        return
                    # raise anything else
                    raise errh
                except requests.exceptions.Timeout as errt:
                    logger.debug("GCS file upload Timeout Error: %s", errt)
                    meta[u'last_error'] = errt
                    meta[u'result_status'] = ResultStatus.NEED_RETRY
                    return

            if meta[u'put_callback']:
                meta[u'put_callback'](
                    data_file,
                    meta[u'src_file_size'],
                    output_stream=meta[u'put_callback_output_stream'],
                    show_progress_bar=meta[u'show_progress_bar'])(
                    meta[u'src_file_size'])

            logger.debug(u'DONE putting a file')
            meta[u'dst_file_size'] = meta[u'upload_size']
            meta[u'result_status'] = ResultStatus.UPLOADED

            meta[GCS_FILE_HEADER_DIGEST] = gcs_headers[GCS_METADATA_SFC_DIGEST]
            meta[GCS_FILE_HEADER_CONTENT_LENGTH] = meta[u'upload_size']
            meta[GCS_FILE_HEADER_ENCRYPTION_METADATA] = \
                gcs_headers.get(GCS_METADATA_ENCRYPTIONDATAPROP, None)
        else:
            # Use the storage client to upload the object
            # Currently not supported, given that we can't obtain
            # location-scoped access tokens
            logger.error(u"GCS upload operation with an access token is "
                         u"currently unsupported")
            meta[u'result_status'] = ResultStatus.ERROR

        Uploads the local file to remote storage
        :param data_file: file path on local system
        :param meta: file meta object (contains credentials and remote location)
        :param encryption_metadata: encryption metadata to be set on object
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        
Uploads the local file to remote storage
Uploads the local file to remote storage
def upload_file(data_file, meta, encryption_metadata, max_concurrency):
        logger = getLogger(__name__)
        if meta.get(u'presigned_url', None):
            content_encoding = ""
            if meta.get(u'dst_compression_type') is not None:
                content_encoding = meta[u'dst_compression_type'][u'name'].lower()
            if content_encoding and content_encoding == u'gzip':
                content_encoding = ""
            gcs_headers = {
                HTTP_HEADER_CONTENT_ENCODING: content_encoding,
                GCS_METADATA_SFC_DIGEST: meta[SHA256_DIGEST],
            }
            if encryption_metadata:
                gcs_headers.update({
                    GCS_METADATA_ENCRYPTIONDATAPROP: json.dumps({
                        u'EncryptionMode': u'FullBlob',
                        u'WrappedContentKey': {
                            u'KeyId': u'symmKey1',
                            u'EncryptedKey': encryption_metadata.key,
                            u'Algorithm': u'AES_CBC_256'
                        },
                        u'EncryptionAgent': {
                            u'Protocol': '1.0',
                            u'EncryptionAlgorithm': u'AES_CBC_256',
                        },
                        u'ContentEncryptionIV': encryption_metadata.iv,
                        u'KeyWrappingMetadata': {
                            u'EncryptionLibrary': u'Java 5.3.0'
                        }
                    }),
                    GCS_METADATA_MATDESC_KEY: encryption_metadata.matdesc
                })
            with open(data_file, 'rb') as fd:
                try:
                    response = requests.put(
                        meta[u'presigned_url'],
                        data=fd,
                        headers=gcs_headers)
                    response.raise_for_status()
                except requests.exceptions.HTTPError as errh:
                    logger.debug("GCS file upload Http Error: %s", errh)
                    if errh.response.status_code in [403, 408, 429, 500, 503]:
                        meta[u'last_error'] = errh
                        meta[u'result_status'] = ResultStatus.NEED_RETRY
                        return
                    elif (errh.response.status_code == 400 and
                          (u'last_error' not in meta or meta[u'last_error'].response.status_code != 400)):
                        meta[u'last_error'] = errh
                        meta[u'result_status'] = ResultStatus.RENEW_PRESIGNED_URL
                        return
                    raise errh
                except requests.exceptions.Timeout as errt:
                    logger.debug("GCS file upload Timeout Error: %s", errt)
                    meta[u'last_error'] = errt
                    meta[u'result_status'] = ResultStatus.NEED_RETRY
                    return
            if meta[u'put_callback']:
                meta[u'put_callback'](
                    data_file,
                    meta[u'src_file_size'],
                    output_stream=meta[u'put_callback_output_stream'],
                    show_progress_bar=meta[u'show_progress_bar'])(
                    meta[u'src_file_size'])
            logger.debug(u'DONE putting a file')
            meta[u'dst_file_size'] = meta[u'upload_size']
            meta[u'result_status'] = ResultStatus.UPLOADED
            meta[GCS_FILE_HEADER_DIGEST] = gcs_headers[GCS_METADATA_SFC_DIGEST]
            meta[GCS_FILE_HEADER_CONTENT_LENGTH] = meta[u'upload_size']
            meta[GCS_FILE_HEADER_ENCRYPTION_METADATA] = \
                gcs_headers.get(GCS_METADATA_ENCRYPTIONDATAPROP, None)
        else:
            logger.error(u"GCS upload operation with an access token is "
                         u"currently unsupported")
            meta[u'result_status'] = ResultStatus.ERROR
def upload_file ( data_file , meta , encryption_metadata , max_concurrency ) : logger = getLogger ( __name__ ) if meta . get ( u'presigned_url' , None ) : content_encoding = "" if meta . get ( u'dst_compression_type' ) is not None : content_encoding = meta [ u'dst_compression_type' ] [ u'name' ] . lower ( ) if content_encoding and content_encoding == u'gzip' : content_encoding = "" gcs_headers = { HTTP_HEADER_CONTENT_ENCODING : content_encoding , GCS_METADATA_SFC_DIGEST : meta [ SHA256_DIGEST ] , } if encryption_metadata : gcs_headers . update ( { GCS_METADATA_ENCRYPTIONDATAPROP : json . dumps ( { u'EncryptionMode' : u'FullBlob' , u'WrappedContentKey' : { u'KeyId' : u'symmKey1' , u'EncryptedKey' : encryption_metadata . key , u'Algorithm' : u'AES_CBC_256' } , u'EncryptionAgent' : { u'Protocol' : '1.0' , u'EncryptionAlgorithm' : u'AES_CBC_256' , } , u'ContentEncryptionIV' : encryption_metadata . iv , u'KeyWrappingMetadata' : { u'EncryptionLibrary' : u'Java 5.3.0' } } ) , GCS_METADATA_MATDESC_KEY : encryption_metadata . matdesc } ) with open ( data_file , 'rb' ) as fd : try : response = requests . put ( meta [ u'presigned_url' ] , data = fd , headers = gcs_headers ) response . raise_for_status ( ) except requests . exceptions . HTTPError as errh : logger . debug ( "GCS file upload Http Error: %s" , errh ) if errh . response . status_code in [ 403 , 408 , 429 , 500 , 503 ] : meta [ u'last_error' ] = errh meta [ u'result_status' ] = ResultStatus . NEED_RETRY return elif ( errh . response . status_code == 400 and ( u'last_error' not in meta or meta [ u'last_error' ] . response . status_code != 400 ) ) : meta [ u'last_error' ] = errh meta [ u'result_status' ] = ResultStatus . RENEW_PRESIGNED_URL return raise errh except requests . exceptions . Timeout as errt : logger . debug ( "GCS file upload Timeout Error: %s" , errt ) meta [ u'last_error' ] = errt meta [ u'result_status' ] = ResultStatus . NEED_RETRY return if meta [ u'put_callback' ] : meta [ u'put_callback' ] ( data_file , meta [ u'src_file_size' ] , output_stream = meta [ u'put_callback_output_stream' ] , show_progress_bar = meta [ u'show_progress_bar' ] ) ( meta [ u'src_file_size' ] ) logger . debug ( u'DONE putting a file' ) meta [ u'dst_file_size' ] = meta [ u'upload_size' ] meta [ u'result_status' ] = ResultStatus . UPLOADED meta [ GCS_FILE_HEADER_DIGEST ] = gcs_headers [ GCS_METADATA_SFC_DIGEST ] meta [ GCS_FILE_HEADER_CONTENT_LENGTH ] = meta [ u'upload_size' ] meta [ GCS_FILE_HEADER_ENCRYPTION_METADATA ] = gcs_headers . get ( GCS_METADATA_ENCRYPTIONDATAPROP , None ) else : logger . error ( u"GCS upload operation with an access token is " u"currently unsupported" ) meta [ u'result_status' ] = ResultStatus . ERROR
Uploads the local file to remote storage
Uploads the local file to remote storage
"""
        Uploads the local file to remote storage
        :param data_file: file path on local system
        :param meta: file meta object (contains credentials and remote location)
        :param encryption_metadata: encryption metadata to be set on object
        :param max_concurrency: (inapplicable to GCS)
        :return: None, if successful. Otherwise, throws Exception.
        """ # Use presigned url to upload the object # We set the contentEncoding to blank for GZIP files. We don't # want GCS to think our gzip files are gzips because it makes # them download uncompressed, and none of the other providers do # that. There's essentially no way for us to prevent that # behavior. Bad Google. # Presigned urls can be generated for any xml-api operation # offered by GCS. Hence the error codes expected are similar # to xml api. # https://cloud.google.com/storage/docs/xml-api/reference-status # According to the above resource, GCS recommends retrying # for the following error codes. # Only attempt to renew urls if this isn't the second time this happens # raise anything else # Use the storage client to upload the object # Currently not supported, given that we can't obtain # location-scoped access tokens
None, if successful. Otherwise, throws Exception.
None if successful . Otherwise throws Exception .
data_file
file path on local system
file path on local system
meta
file meta object (contains credentials and remote location)
file meta object ( contains credentials and remote location )
encryption_metadata
encryption metadata to be set on object
encryption metadata to be set on object
max_concurrency
(inapplicable to GCS)
( inapplicable to GCS )