labpack/storage/aws/s3.py
def create_record(self, bucket_name, record_key, record_data, record_metadata=None, record_mimetype='', record_encoding='', overwrite=True):

        '''
            a method for adding a record to an S3 bucket
            
        :param bucket_name: string with name of bucket
        :param record_key: string with name of key (path) for record
        :param record_data: byte data for record
        :param record_metadata: [optional] dictionary with metadata to attach to record
        :param record_mimetype: [optional] string with content mimetype of record data
        :param record_encoding: [optional] string with content encoding of record data
        :param overwrite: [optional] boolean to overwrite any existing record
        :return: string with name of record key
        '''
        
        title = '%s.create_record' % self.__class__.__name__

        import sys
        from hashlib import md5
        from base64 import b64encode
    
    # define size limitations
        metadata_max = self.fields.metadata['limits']['metadata_max_bytes']
        record_max = self.fields.metadata['limits']['record_max_bytes']
        record_optimal = self.fields.metadata['limits']['record_optimal_bytes']
        
    # validate inputs
        input_fields = {
            'bucket_name': bucket_name,
            'record_key': record_key,
            'record_metadata': record_metadata,
            'secret_key': secret_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
    
    # validate metadata fields
        if record_metadata:
            for key, value in record_metadata.items():
                object_title = '%s(record_metadata={%s:...})' % (title, key)
                self.fields.validate(key, '.metadata_keys', object_title)
                object_title = '%s(record_metadata={%s:%s})' % (title, key, value)
                self.fields.validate(value, '.metadata_values', object_title)
        else:
            record_metadata = {}
    
    # verify existence of bucket
        if not bucket_name in self.bucket_list:
            if not bucket_name in self.list_buckets():
                raise ValueError('S3 bucket "%s" does not exist in aws region %s.' % (bucket_name, self.iam.region_name))
    
    # verify overwrite condition
        if not overwrite:
            record_status = self.read_headers(bucket_name, record_key)
            if record_status:
                error_msg = 'S3 bucket "%s" already contains a record for key "%s"' % (bucket_name, record_key)
                raise ValueError(error_msg)
        
    # validate size of metadata
        import json
        metadata_size = sys.getsizeof(json.dumps(record_metadata).encode('utf-8'))
        if metadata_size > metadata_max:
            raise ValueError('%s(record_metadata={...}) cannot be greater than %s bytes.' % (title, metadata_max))
                
    # validate size of record
        record_size = sys.getsizeof(record_data)
        error_prefix = '%s(record_key="%s", record_data=...)' % (title, record_key)
        if record_size > record_max:
            raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
        elif record_size > record_optimal:
            self.iam.printer('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))

    # determine content encoding
        if not record_encoding or not record_mimetype:
            import mimetypes
            guess_mimetype, guess_encoding = mimetypes.guess_type(record_key)
            if not record_mimetype:
                record_mimetype = guess_mimetype
            if not record_encoding:
                record_encoding = guess_encoding
                    
    # create RFC 1864 base64 encoded md5 digest of data to confirm integrity
        hash_bytes = md5(record_data).digest()
        content_md5 = b64encode(hash_bytes).decode()
        check_sum = md5(record_data).hexdigest()

    # create key word argument dictionary
        create_kwargs = {
            'Bucket': bucket_name,
            'Key': record_key,
            'Body': record_data,
            'ContentMD5': content_md5
        }
        if record_metadata:
            create_kwargs['Metadata'] = record_metadata
        if record_mimetype:
            create_kwargs['ContentType'] = record_mimetype
        if record_encoding:
            create_kwargs['ContentEncoding'] = record_encoding

    # add server side encryption key to request
    #     if server_encryption:
    #         key_b64, digest_b64 = self._key_digest(secret_key)
    #         kw_args['SSECustomerAlgorithm'] = 'AES256'
    #         kw_args['SSECustomerKey'] = key_b64
    #         kw_args['SSECustomerKeyMD5'] = digest_b64

    # send request to add object
        try:
            response = self.connection.put_object(**create_kwargs)
            if response['ETag'] != '"%s"' % check_sum:
                raise Exception('%s experienced a data transcription error for record "%s".' % record_key)
        except:
            raise AWSConnectionError(title)

        return record_key

            a method for adding a record to an S3 bucket
            
        :param bucket_name: string with name of bucket
        :param record_key: string with name of key (path) for record
        :param record_data: byte data for record
        :param record_metadata: [optional] dictionary with metadata to attach to record
        :param record_mimetype: [optional] string with content mimetype of record data
        :param record_encoding: [optional] string with content encoding of record data
        :param overwrite: [optional] boolean to overwrite any existing record
        :return: string with name of record key
        
a method for adding a record to an S3 bucket
a method for adding a record to an S3 bucket
def create_record(self, bucket_name, record_key, record_data, record_metadata=None, record_mimetype='', record_encoding='', overwrite=True):
        title = '%s.create_record' % self.__class__.__name__
        import sys
        from hashlib import md5
        from base64 import b64encode
        metadata_max = self.fields.metadata['limits']['metadata_max_bytes']
        record_max = self.fields.metadata['limits']['record_max_bytes']
        record_optimal = self.fields.metadata['limits']['record_optimal_bytes']
        input_fields = {
            'bucket_name': bucket_name,
            'record_key': record_key,
            'record_metadata': record_metadata,
            'secret_key': secret_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
        if record_metadata:
            for key, value in record_metadata.items():
                object_title = '%s(record_metadata={%s:...})' % (title, key)
                self.fields.validate(key, '.metadata_keys', object_title)
                object_title = '%s(record_metadata={%s:%s})' % (title, key, value)
                self.fields.validate(value, '.metadata_values', object_title)
        else:
            record_metadata = {}
        if not bucket_name in self.bucket_list:
            if not bucket_name in self.list_buckets():
                raise ValueError('S3 bucket "%s" does not exist in aws region %s.' % (bucket_name, self.iam.region_name))
        if not overwrite:
            record_status = self.read_headers(bucket_name, record_key)
            if record_status:
                error_msg = 'S3 bucket "%s" already contains a record for key "%s"' % (bucket_name, record_key)
                raise ValueError(error_msg)
        import json
        metadata_size = sys.getsizeof(json.dumps(record_metadata).encode('utf-8'))
        if metadata_size > metadata_max:
            raise ValueError('%s(record_metadata={...}) cannot be greater than %s bytes.' % (title, metadata_max))
        record_size = sys.getsizeof(record_data)
        error_prefix = '%s(record_key="%s", record_data=...)' % (title, record_key)
        if record_size > record_max:
            raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
        elif record_size > record_optimal:
            self.iam.printer('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))
        if not record_encoding or not record_mimetype:
            import mimetypes
            guess_mimetype, guess_encoding = mimetypes.guess_type(record_key)
            if not record_mimetype:
                record_mimetype = guess_mimetype
            if not record_encoding:
                record_encoding = guess_encoding
        hash_bytes = md5(record_data).digest()
        content_md5 = b64encode(hash_bytes).decode()
        check_sum = md5(record_data).hexdigest()
        create_kwargs = {
            'Bucket': bucket_name,
            'Key': record_key,
            'Body': record_data,
            'ContentMD5': content_md5
        }
        if record_metadata:
            create_kwargs['Metadata'] = record_metadata
        if record_mimetype:
            create_kwargs['ContentType'] = record_mimetype
        if record_encoding:
            create_kwargs['ContentEncoding'] = record_encoding
        try:
            response = self.connection.put_object(**create_kwargs)
            if response['ETag'] != '"%s"' % check_sum:
                raise Exception('%s experienced a data transcription error for record "%s".' % record_key)
        except:
            raise AWSConnectionError(title)
        return record_key
def create_record ( self , bucket_name , record_key , record_data , record_metadata = None , record_mimetype = '' , record_encoding = '' , overwrite = True ) : title = '%s.create_record' % self . __class__ . __name__ import sys from hashlib import md5 from base64 import b64encode metadata_max = self . fields . metadata [ 'limits' ] [ 'metadata_max_bytes' ] record_max = self . fields . metadata [ 'limits' ] [ 'record_max_bytes' ] record_optimal = self . fields . metadata [ 'limits' ] [ 'record_optimal_bytes' ] input_fields = { 'bucket_name' : bucket_name , 'record_key' : record_key , 'record_metadata' : record_metadata , 'secret_key' : secret_key } for key , value in input_fields . items ( ) : if value : object_title = '%s(%s=%s)' % ( title , key , str ( value ) ) self . fields . validate ( value , '.%s' % key , object_title ) if record_metadata : for key , value in record_metadata . items ( ) : object_title = '%s(record_metadata={%s:...})' % ( title , key ) self . fields . validate ( key , '.metadata_keys' , object_title ) object_title = '%s(record_metadata={%s:%s})' % ( title , key , value ) self . fields . validate ( value , '.metadata_values' , object_title ) else : record_metadata = { } if not bucket_name in self . bucket_list : if not bucket_name in self . list_buckets ( ) : raise ValueError ( 'S3 bucket "%s" does not exist in aws region %s.' % ( bucket_name , self . iam . region_name ) ) if not overwrite : record_status = self . read_headers ( bucket_name , record_key ) if record_status : error_msg = 'S3 bucket "%s" already contains a record for key "%s"' % ( bucket_name , record_key ) raise ValueError ( error_msg ) import json metadata_size = sys . getsizeof ( json . dumps ( record_metadata ) . encode ( 'utf-8' ) ) if metadata_size > metadata_max : raise ValueError ( '%s(record_metadata={...}) cannot be greater than %s bytes.' % ( title , metadata_max ) ) record_size = sys . getsizeof ( record_data ) error_prefix = '%s(record_key="%s", record_data=...)' % ( title , record_key ) if record_size > record_max : raise ValueError ( '%s exceeds maximum record data size of %s bytes.' % ( error_prefix , record_max ) ) elif record_size > record_optimal : self . iam . printer ( '[WARNING] %s exceeds optimal record data size of %s bytes.' % ( error_prefix , record_optimal ) ) if not record_encoding or not record_mimetype : import mimetypes guess_mimetype , guess_encoding = mimetypes . guess_type ( record_key ) if not record_mimetype : record_mimetype = guess_mimetype if not record_encoding : record_encoding = guess_encoding hash_bytes = md5 ( record_data ) . digest ( ) content_md5 = b64encode ( hash_bytes ) . decode ( ) check_sum = md5 ( record_data ) . hexdigest ( ) create_kwargs = { 'Bucket' : bucket_name , 'Key' : record_key , 'Body' : record_data , 'ContentMD5' : content_md5 } if record_metadata : create_kwargs [ 'Metadata' ] = record_metadata if record_mimetype : create_kwargs [ 'ContentType' ] = record_mimetype if record_encoding : create_kwargs [ 'ContentEncoding' ] = record_encoding try : response = self . connection . put_object ( ** create_kwargs ) if response [ 'ETag' ] != '"%s"' % check_sum : raise Exception ( '%s experienced a data transcription error for record "%s".' % record_key ) except : raise AWSConnectionError ( title ) return record_key
a method for adding a record to an S3 bucket
a method for adding a record to an S3 bucket
'''
            a method for adding a record to an S3 bucket
            
        :param bucket_name: string with name of bucket
        :param record_key: string with name of key (path) for record
        :param record_data: byte data for record
        :param record_metadata: [optional] dictionary with metadata to attach to record
        :param record_mimetype: [optional] string with content mimetype of record data
        :param record_encoding: [optional] string with content encoding of record data
        :param overwrite: [optional] boolean to overwrite any existing record
        :return: string with name of record key
        ''' # define size limitations # validate inputs # validate metadata fields # verify existence of bucket # verify overwrite condition # validate size of metadata # validate size of record # determine content encoding # create RFC 1864 base64 encoded md5 digest of data to confirm integrity # create key word argument dictionary # add server side encryption key to request #     if server_encryption: #         key_b64, digest_b64 = self._key_digest(secret_key) #         kw_args['SSECustomerAlgorithm'] = 'AES256' #         kw_args['SSECustomerKey'] = key_b64 #         kw_args['SSECustomerKeyMD5'] = digest_b64 # send request to add object
string with name of record key
string with name of record key
self
bucket_name
string with name of bucket
string with name of bucket
record_key
string with name of key (path) for record
string with name of key ( path ) for record
record_data
byte data for record
byte data for record
record_metadata
[optional] dictionary with metadata to attach to record
[ optional ] dictionary with metadata to attach to record
record_mimetype
[optional] string with content mimetype of record data
[ optional ] string with content mimetype of record data
record_encoding
[optional] string with content encoding of record data
[ optional ] string with content encoding of record data
overwrite
[optional] boolean to overwrite any existing record
[ optional ] boolean to overwrite any existing record