labpack/storage/aws/s3.py
def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
    
        ''' 
            a method to list keys in the collection

        :param prefix: string with prefix value to filter results
        :param delimiter: string with value results must not contain (after prefix)
        :param filter_function: (positional arguments) function used to filter results
        :param max_results: integer with maximum number of results to return
        :param previous_key: string with key in collection to begin search after
        :return: list of key strings
        
        NOTE:   each key string can be divided into one or more segments
                based upon the / characters which occur in the key string as
                well as its file extension type. if the key string represents
                a file path, then each directory in the path, the file name
                and the file extension are all separate indexed values.

                eg. lab/unittests/1473719695.2165067.json is indexed:
                [ 'lab', 'unittests', '1473719695.2165067', '.json' ]

                it is possible to filter the records in the collection according
                to one or more of these path segments using a filter_function.

        NOTE:   the filter_function must be able to accept an array of positional
                arguments and return a value that can evaluate to true or false.
                while searching the records, list produces an array of strings
                which represent the directory structure in relative path of each
                key string. if a filter_function is provided, this list of strings
                is fed to the filter function. if the function evaluates this input
                and returns a true value the file will be included in the list
                results.
        '''
        
        title = '%s.list' % self.__class__.__name__

    # validate input
        input_fields = {
            'prefix': prefix,
            'delimiter': delimiter,
            'max_results': max_results,
            'record_key': previous_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
    
    # construct default response
        results_list = []
        
    # handle filter function filter
        if filter_function:
        
        # validate filter function
            try:
                path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
                filter_function(*path_segments)
            except:
                err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
                raise TypeError('%s must accept positional arguments.' % err_msg)
            
        # construct keyword arguments
            list_kwargs = {
                'bucket_name': self.bucket_name,
                'prefix': prefix,
                'delimiter': delimiter
            }
            
        # determine starting key
            starting_key = '1'
            if previous_key:
                previous_kwargs = {}
                previous_kwargs.update(**list_kwargs)
                previous_kwargs['max_results'] = 1
                previous_kwargs['starting_key'] = previous_key
                search_list, next_key = self.s3.list_records(**list_kwargs)
                list_kwargs['starting_key'] = next_key
        
        # iterate filter over collection
            import os
            while starting_key:
                search_list, starting_key = self.s3.list_records(**list_kwargs)
                for record in search_list:
                    record_key = record['key']
                    path_segments = record_key.split(os.sep)
                    if filter_function(*path_segments):
                        results_list.append(record_key)
                    if len(results_list) == max_results:
                        return results_list
        
    # handle other filters
        else:
            
        # construct keyword arguments
            list_kwargs = {
                'bucket_name': self.bucket_name,
                'prefix': prefix,
                'delimiter': delimiter,
                'max_results': max_results
            }
        
        # determine starting key
            if previous_key:
                previous_kwargs = {}
                previous_kwargs.update(**list_kwargs)
                previous_kwargs['max_results'] = 1
                previous_kwargs['starting_key'] = previous_key
                search_list, starting_key = self.s3.list_records(**list_kwargs)
                list_kwargs['starting_key'] = starting_key
    
        # retrieve results 
            search_list, starting_key = self.s3.list_records(**list_kwargs)
    
        # construct result list
            for record in search_list:
                results_list.append(record['key'])
        
        return results_list
 
            a method to list keys in the collection

        :param prefix: string with prefix value to filter results
        :param delimiter: string with value results must not contain (after prefix)
        :param filter_function: (positional arguments) function used to filter results
        :param max_results: integer with maximum number of results to return
        :param previous_key: string with key in collection to begin search after
        :return: list of key strings
        
        NOTE:   each key string can be divided into one or more segments
                based upon the / characters which occur in the key string as
                well as its file extension type. if the key string represents
                a file path, then each directory in the path, the file name
                and the file extension are all separate indexed values.

                eg. lab/unittests/1473719695.2165067.json is indexed:
                [ 'lab', 'unittests', '1473719695.2165067', '.json' ]

                it is possible to filter the records in the collection according
                to one or more of these path segments using a filter_function.

        NOTE:   the filter_function must be able to accept an array of positional
                arguments and return a value that can evaluate to true or false.
                while searching the records, list produces an array of strings
                which represent the directory structure in relative path of each
                key string. if a filter_function is provided, this list of strings
                is fed to the filter function. if the function evaluates this input
                and returns a true value the file will be included in the list
                results.
        
a method to list keys in the collection
a method to list keys in the collection
def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
        title = '%s.list' % self.__class__.__name__
        input_fields = {
            'prefix': prefix,
            'delimiter': delimiter,
            'max_results': max_results,
            'record_key': previous_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
        results_list = []
        if filter_function:
            try:
                path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
                filter_function(*path_segments)
            except:
                err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
                raise TypeError('%s must accept positional arguments.' % err_msg)
            list_kwargs = {
                'bucket_name': self.bucket_name,
                'prefix': prefix,
                'delimiter': delimiter
            }
            starting_key = '1'
            if previous_key:
                previous_kwargs = {}
                previous_kwargs.update(**list_kwargs)
                previous_kwargs['max_results'] = 1
                previous_kwargs['starting_key'] = previous_key
                search_list, next_key = self.s3.list_records(**list_kwargs)
                list_kwargs['starting_key'] = next_key
            import os
            while starting_key:
                search_list, starting_key = self.s3.list_records(**list_kwargs)
                for record in search_list:
                    record_key = record['key']
                    path_segments = record_key.split(os.sep)
                    if filter_function(*path_segments):
                        results_list.append(record_key)
                    if len(results_list) == max_results:
                        return results_list
        else:
            list_kwargs = {
                'bucket_name': self.bucket_name,
                'prefix': prefix,
                'delimiter': delimiter,
                'max_results': max_results
            }
            if previous_key:
                previous_kwargs = {}
                previous_kwargs.update(**list_kwargs)
                previous_kwargs['max_results'] = 1
                previous_kwargs['starting_key'] = previous_key
                search_list, starting_key = self.s3.list_records(**list_kwargs)
                list_kwargs['starting_key'] = starting_key
            search_list, starting_key = self.s3.list_records(**list_kwargs)
            for record in search_list:
                results_list.append(record['key'])
        return results_list
def list ( self , prefix = '' , delimiter = '' , filter_function = None , max_results = 1 , previous_key = '' ) : title = '%s.list' % self . __class__ . __name__ input_fields = { 'prefix' : prefix , 'delimiter' : delimiter , 'max_results' : max_results , 'record_key' : previous_key } for key , value in input_fields . items ( ) : if value : object_title = '%s(%s=%s)' % ( title , key , str ( value ) ) self . fields . validate ( value , '.%s' % key , object_title ) results_list = [ ] if filter_function : try : path_segments = [ 'lab' , 'unittests' , '1473719695.2165067' , '.json' ] filter_function ( * path_segments ) except : err_msg = '%s(filter_function=%s)' % ( title , filter_function . __class__ . __name__ ) raise TypeError ( '%s must accept positional arguments.' % err_msg ) list_kwargs = { 'bucket_name' : self . bucket_name , 'prefix' : prefix , 'delimiter' : delimiter } starting_key = '1' if previous_key : previous_kwargs = { } previous_kwargs . update ( ** list_kwargs ) previous_kwargs [ 'max_results' ] = 1 previous_kwargs [ 'starting_key' ] = previous_key search_list , next_key = self . s3 . list_records ( ** list_kwargs ) list_kwargs [ 'starting_key' ] = next_key import os while starting_key : search_list , starting_key = self . s3 . list_records ( ** list_kwargs ) for record in search_list : record_key = record [ 'key' ] path_segments = record_key . split ( os . sep ) if filter_function ( * path_segments ) : results_list . append ( record_key ) if len ( results_list ) == max_results : return results_list else : list_kwargs = { 'bucket_name' : self . bucket_name , 'prefix' : prefix , 'delimiter' : delimiter , 'max_results' : max_results } if previous_key : previous_kwargs = { } previous_kwargs . update ( ** list_kwargs ) previous_kwargs [ 'max_results' ] = 1 previous_kwargs [ 'starting_key' ] = previous_key search_list , starting_key = self . s3 . list_records ( ** list_kwargs ) list_kwargs [ 'starting_key' ] = starting_key search_list , starting_key = self . s3 . list_records ( ** list_kwargs ) for record in search_list : results_list . append ( record [ 'key' ] ) return results_list
a method to list keys in the collection
a method to list keys in the collection
''' 
            a method to list keys in the collection

        :param prefix: string with prefix value to filter results
        :param delimiter: string with value results must not contain (after prefix)
        :param filter_function: (positional arguments) function used to filter results
        :param max_results: integer with maximum number of results to return
        :param previous_key: string with key in collection to begin search after
        :return: list of key strings
        
        NOTE:   each key string can be divided into one or more segments
                based upon the / characters which occur in the key string as
                well as its file extension type. if the key string represents
                a file path, then each directory in the path, the file name
                and the file extension are all separate indexed values.

                eg. lab/unittests/1473719695.2165067.json is indexed:
                [ 'lab', 'unittests', '1473719695.2165067', '.json' ]

                it is possible to filter the records in the collection according
                to one or more of these path segments using a filter_function.

        NOTE:   the filter_function must be able to accept an array of positional
                arguments and return a value that can evaluate to true or false.
                while searching the records, list produces an array of strings
                which represent the directory structure in relative path of each
                key string. if a filter_function is provided, this list of strings
                is fed to the filter function. if the function evaluates this input
                and returns a true value the file will be included in the list
                results.
        ''' # validate input # construct default response # handle filter function filter # validate filter function # construct keyword arguments # determine starting key # iterate filter over collection # handle other filters # construct keyword arguments # determine starting key # retrieve results  # construct result list
list of key strings
NOTE:   each key string can be divided into one or more segments
based upon the / characters which occur in the key string as
well as its file extension type. if the key string represents
a file path, then each directory in the path, the file name
and the file extension are all separate indexed values.

eg.

it is possible to filter the records in the collection according
to one or more of these path segments using a filter_function.

the filter_function must be able to accept an array of positional
arguments and return a value that can evaluate to true or false.
while searching the records, list produces an array of strings
which represent the directory structure in relative path of each
key string. if a filter_function is provided, this list of strings
is fed to the filter function. if the function evaluates this input
and returns a true value the file will be included in the list
results.
list of key strings NOTE : each key string can be divided into one or more segments based upon the / characters which occur in the key string as well as its file extension type . if the key string represents a file path then each directory in the path the file name and the file extension are all separate indexed values . eg . it is possible to filter the records in the collection according to one or more of these path segments using a filter_function . the filter_function must be able to accept an array of positional arguments and return a value that can evaluate to true or false . while searching the records list produces an array of strings which represent the directory structure in relative path of each key string . if a filter_function is provided this list of strings is fed to the filter function . if the function evaluates this input and returns a true value the file will be included in the list results .
self
prefix
string with prefix value to filter results
string with prefix value to filter results
delimiter
string with value results must not contain (after prefix)
string with value results must not contain ( after prefix )
filter_function
(positional arguments) function used to filter results
( positional arguments ) function used to filter results
max_results
integer with maximum number of results to return
integer with maximum number of results to return
previous_key
string with key in collection to begin search after
string with key in collection to begin search after