labpack/storage/aws/s3.py
def export_records(self, bucket_name, export_path='', overwrite=True):

        '''
            a method to export all the records from a bucket to local files

        :param bucket_name: string with name of bucket
        :param export_path: [optional] string with path to root directory for record dump
        :param overwrite: [optional] boolean to overwrite existing files matching records
        :return: True
        '''

        title = '%s.export_records' % self.__class__.__name__
        
    # validate inputs
        input_fields = {
            'bucket_name': bucket_name,
            'export_path': export_path
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
        
    # validate path
        from os import path, makedirs
        if not export_path:
            export_path = './'
        if not path.exists(export_path):
            raise ValueError('%s(export_path="%s") is not a valid path.' % (title, export_path))
        elif not path.isdir(export_path):
            raise ValueError('%s(export_path="%s") must be a directory.' % (title, export_path))
    
    # verify existence of bucket
        if not bucket_name in self.bucket_list:
            if not bucket_name in self.list_buckets():
                raise ValueError('S3 bucket "%s" does not exist in aws region %s.' % (bucket_name, self.iam.region_name))

    # retrieve list of records in bucket
        record_list, next_key = self.list_records(bucket_name)
        if next_key:
            record_number = 'first %s' % str(len(record_list))
        else:
            record_number = str(len(record_list))
        plural = ''
        if len(record_list) != 1:
            plural = 's'
        self.iam.printer('Exporting %s record%s from bucket "%s" to path "%s"' % (record_number, plural, bucket_name, export_path), flush=True)

    # define local save function
        def save_to_file(_export_path, _bucket_name, _record_key, _overwrite):
            
            try:
                response = self.connection.get_object(Bucket=_bucket_name,Key=_record_key)
            except:
                raise AWSConnectionError(title)
            record_data = response['Body'].read()
            file_path = path.join(_export_path, _record_key)
            dir_path = path.dirname(file_path)
            if not path.exists(dir_path):
                makedirs(dir_path)
            if path.exists(file_path) and not _overwrite:
                self.iam.printer('.\n%s already exists. File skipped. Continuing.' % file_path, flush=True)
            else:
                with open(file_path, 'wb') as file:
                    file.write(record_data)
                    file.close()
                self.iam.printer('.', flush=True)
            
    # retrieve data for records in bucket
        for record in record_list:
            save_to_file(export_path, bucket_name, record['key'], overwrite)

    # continue exporting records in bucket until all exported
        if next_key:
            while next_key or record_list:
                record_list, next_key = self.list_records(bucket_name)
                if next_key:
                    record_number = 'next %s' % str(len(record_list))
                else:
                    record_number = 'last %s' % str(len(record_list))
                self.iam.printer('.')
                plural = ''
                if len(record_list) != 1:
                    plural = 's'
                self.iam.printer('Exporting %s record%s from bucket "%s" to path "%s"' % (record_number, plural, bucket_name, export_path), flush=True)
                for record in record_list:
                    save_to_file(export_path, bucket_name, record['key'], overwrite)

    # report completion and return true
        self.iam.printer(' done.')
        return True

            a method to export all the records from a bucket to local files

        :param bucket_name: string with name of bucket
        :param export_path: [optional] string with path to root directory for record dump
        :param overwrite: [optional] boolean to overwrite existing files matching records
        :return: True
        
a method to export all the records from a bucket to local files
a method to export all the records from a bucket to local files
def export_records(self, bucket_name, export_path='', overwrite=True):
        title = '%s.export_records' % self.__class__.__name__
        input_fields = {
            'bucket_name': bucket_name,
            'export_path': export_path
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
        from os import path, makedirs
        if not export_path:
            export_path = './'
        if not path.exists(export_path):
            raise ValueError('%s(export_path="%s") is not a valid path.' % (title, export_path))
        elif not path.isdir(export_path):
            raise ValueError('%s(export_path="%s") must be a directory.' % (title, export_path))
        if not bucket_name in self.bucket_list:
            if not bucket_name in self.list_buckets():
                raise ValueError('S3 bucket "%s" does not exist in aws region %s.' % (bucket_name, self.iam.region_name))
        record_list, next_key = self.list_records(bucket_name)
        if next_key:
            record_number = 'first %s' % str(len(record_list))
        else:
            record_number = str(len(record_list))
        plural = ''
        if len(record_list) != 1:
            plural = 's'
        self.iam.printer('Exporting %s record%s from bucket "%s" to path "%s"' % (record_number, plural, bucket_name, export_path), flush=True)
        def save_to_file(_export_path, _bucket_name, _record_key, _overwrite):
            try:
                response = self.connection.get_object(Bucket=_bucket_name,Key=_record_key)
            except:
                raise AWSConnectionError(title)
            record_data = response['Body'].read()
            file_path = path.join(_export_path, _record_key)
            dir_path = path.dirname(file_path)
            if not path.exists(dir_path):
                makedirs(dir_path)
            if path.exists(file_path) and not _overwrite:
                self.iam.printer('.\n%s already exists. File skipped. Continuing.' % file_path, flush=True)
            else:
                with open(file_path, 'wb') as file:
                    file.write(record_data)
                    file.close()
                self.iam.printer('.', flush=True)
        for record in record_list:
            save_to_file(export_path, bucket_name, record['key'], overwrite)
        if next_key:
            while next_key or record_list:
                record_list, next_key = self.list_records(bucket_name)
                if next_key:
                    record_number = 'next %s' % str(len(record_list))
                else:
                    record_number = 'last %s' % str(len(record_list))
                self.iam.printer('.')
                plural = ''
                if len(record_list) != 1:
                    plural = 's'
                self.iam.printer('Exporting %s record%s from bucket "%s" to path "%s"' % (record_number, plural, bucket_name, export_path), flush=True)
                for record in record_list:
                    save_to_file(export_path, bucket_name, record['key'], overwrite)
        self.iam.printer(' done.')
        return True
def export_records ( self , bucket_name , export_path = '' , overwrite = True ) : title = '%s.export_records' % self . __class__ . __name__ input_fields = { 'bucket_name' : bucket_name , 'export_path' : export_path } for key , value in input_fields . items ( ) : if value : object_title = '%s(%s=%s)' % ( title , key , str ( value ) ) self . fields . validate ( value , '.%s' % key , object_title ) from os import path , makedirs if not export_path : export_path = './' if not path . exists ( export_path ) : raise ValueError ( '%s(export_path="%s") is not a valid path.' % ( title , export_path ) ) elif not path . isdir ( export_path ) : raise ValueError ( '%s(export_path="%s") must be a directory.' % ( title , export_path ) ) if not bucket_name in self . bucket_list : if not bucket_name in self . list_buckets ( ) : raise ValueError ( 'S3 bucket "%s" does not exist in aws region %s.' % ( bucket_name , self . iam . region_name ) ) record_list , next_key = self . list_records ( bucket_name ) if next_key : record_number = 'first %s' % str ( len ( record_list ) ) else : record_number = str ( len ( record_list ) ) plural = '' if len ( record_list ) != 1 : plural = 's' self . iam . printer ( 'Exporting %s record%s from bucket "%s" to path "%s"' % ( record_number , plural , bucket_name , export_path ) , flush = True ) def save_to_file ( _export_path , _bucket_name , _record_key , _overwrite ) : try : response = self . connection . get_object ( Bucket = _bucket_name , Key = _record_key ) except : raise AWSConnectionError ( title ) record_data = response [ 'Body' ] . read ( ) file_path = path . join ( _export_path , _record_key ) dir_path = path . dirname ( file_path ) if not path . exists ( dir_path ) : makedirs ( dir_path ) if path . exists ( file_path ) and not _overwrite : self . iam . printer ( '.\n%s already exists. File skipped. Continuing.' % file_path , flush = True ) else : with open ( file_path , 'wb' ) as file : file . write ( record_data ) file . close ( ) self . iam . printer ( '.' , flush = True ) for record in record_list : save_to_file ( export_path , bucket_name , record [ 'key' ] , overwrite ) if next_key : while next_key or record_list : record_list , next_key = self . list_records ( bucket_name ) if next_key : record_number = 'next %s' % str ( len ( record_list ) ) else : record_number = 'last %s' % str ( len ( record_list ) ) self . iam . printer ( '.' ) plural = '' if len ( record_list ) != 1 : plural = 's' self . iam . printer ( 'Exporting %s record%s from bucket "%s" to path "%s"' % ( record_number , plural , bucket_name , export_path ) , flush = True ) for record in record_list : save_to_file ( export_path , bucket_name , record [ 'key' ] , overwrite ) self . iam . printer ( ' done.' ) return True
a method to export all the records from a bucket to local files
a method to export all the records from a bucket to local files
'''
            a method to export all the records from a bucket to local files

        :param bucket_name: string with name of bucket
        :param export_path: [optional] string with path to root directory for record dump
        :param overwrite: [optional] boolean to overwrite existing files matching records
        :return: True
        ''' # validate inputs # validate path # verify existence of bucket # retrieve list of records in bucket # define local save function # retrieve data for records in bucket # continue exporting records in bucket until all exported # report completion and return true
None
self
bucket_name
string with name of bucket
string with name of bucket
export_path
[optional] string with path to root directory for record dump
[ optional ] string with path to root directory for record dump
overwrite
[optional] boolean to overwrite existing files matching records
[ optional ] boolean to overwrite existing files matching records