src/dirbs/cli/catalog.py
def _harvest_files(prospectors, logger):
    """
    Traverse all specified prospector paths and determine uncataloged files.

    :param prospectors: file prospectors to harvest files
    :param logger: logger instance
    :return: dict list
    """
    discovered_files = []

    # List of files specified explicitly in the prospectors
    absolute_file_paths = [path for x in prospectors for path in x['paths'] if isfile(path)]
    for prospector in prospectors:
        file_type = prospector['file_type']
        for path in prospector['paths']:
            if isdir(path) and exists(path):
                logger.info('Harvesting {0} files from directory...: {1}'.format(file_type, path))
                files = [join(path, f) for f in listdir(path) if isfile(join(path, f))]
                logger.info('Found {0} {1} file(s) at directory: {2}'.format(len(files), file_type, path))
            elif isfile(path):
                logger.info('Harvesting {0} file from path...: {1}'.format(file_type, path))
                files = [path]
            else:
                logger.error('The path specified for {0} file type does not exist: {1}'
                             .format(file_type, path))
                logger.error('Please verify the correct path was specified in the config and '
                             'that the path is readable from the processing machine.')
                sys.exit(1)
            logger.info('Fetching properties for {0} file(s)...'.format(file_type))
            for file_name in files:
                # If file was specified explicitly in prospector; do not index
                # it from other directory paths. This is done to make sure the
                # right file_type and schema is associated with the file.
                if file_name in absolute_file_paths and file_name != path:
                    continue
                if file_name.lower()[-3:] == 'zip':
                    # Stat the file to fetch the file properties and store the relevant fields
                    file_stat_result = stat(file_name)
                    file_properties = CatalogAttributes(basename(file_name), file_type,
                                                        datetime.utcfromtimestamp(file_stat_result.st_mtime),
                                                        file_stat_result.st_size)
                    discovered_files.append({'file_path': file_name, 'file_properties': file_properties,
                                             'schema': prospector['schema']})
                else:
                    logger.warning('Non-zip file found in path and will be ignored: {0}'.format(file_name))
            logger.info('Finished fetching properties for {0} files'.format(file_type))

    logger.info('Harvested a total of {0} file(s)'.format(len(discovered_files)))
    return discovered_files

    Traverse all specified prospector paths and determine uncataloged files.

    :param prospectors: file prospectors to harvest files
    :param logger: logger instance
    :return: dict list
    
Traverse all specified prospector paths and determine uncataloged files.
Traverse all specified prospector paths and determine uncataloged files .
def _harvest_files(prospectors, logger):
    discovered_files = []
    absolute_file_paths = [path for x in prospectors for path in x['paths'] if isfile(path)]
    for prospector in prospectors:
        file_type = prospector['file_type']
        for path in prospector['paths']:
            if isdir(path) and exists(path):
                logger.info('Harvesting {0} files from directory...: {1}'.format(file_type, path))
                files = [join(path, f) for f in listdir(path) if isfile(join(path, f))]
                logger.info('Found {0} {1} file(s) at directory: {2}'.format(len(files), file_type, path))
            elif isfile(path):
                logger.info('Harvesting {0} file from path...: {1}'.format(file_type, path))
                files = [path]
            else:
                logger.error('The path specified for {0} file type does not exist: {1}'
                             .format(file_type, path))
                logger.error('Please verify the correct path was specified in the config and '
                             'that the path is readable from the processing machine.')
                sys.exit(1)
            logger.info('Fetching properties for {0} file(s)...'.format(file_type))
            for file_name in files:
                if file_name in absolute_file_paths and file_name != path:
                    continue
                if file_name.lower()[-3:] == 'zip':
                    file_stat_result = stat(file_name)
                    file_properties = CatalogAttributes(basename(file_name), file_type,
                                                        datetime.utcfromtimestamp(file_stat_result.st_mtime),
                                                        file_stat_result.st_size)
                    discovered_files.append({'file_path': file_name, 'file_properties': file_properties,
                                             'schema': prospector['schema']})
                else:
                    logger.warning('Non-zip file found in path and will be ignored: {0}'.format(file_name))
            logger.info('Finished fetching properties for {0} files'.format(file_type))
    logger.info('Harvested a total of {0} file(s)'.format(len(discovered_files)))
    return discovered_files
def _harvest_files ( prospectors , logger ) : discovered_files = [ ] absolute_file_paths = [ path for x in prospectors for path in x [ 'paths' ] if isfile ( path ) ] for prospector in prospectors : file_type = prospector [ 'file_type' ] for path in prospector [ 'paths' ] : if isdir ( path ) and exists ( path ) : logger . info ( 'Harvesting {0} files from directory...: {1}' . format ( file_type , path ) ) files = [ join ( path , f ) for f in listdir ( path ) if isfile ( join ( path , f ) ) ] logger . info ( 'Found {0} {1} file(s) at directory: {2}' . format ( len ( files ) , file_type , path ) ) elif isfile ( path ) : logger . info ( 'Harvesting {0} file from path...: {1}' . format ( file_type , path ) ) files = [ path ] else : logger . error ( 'The path specified for {0} file type does not exist: {1}' . format ( file_type , path ) ) logger . error ( 'Please verify the correct path was specified in the config and ' 'that the path is readable from the processing machine.' ) sys . exit ( 1 ) logger . info ( 'Fetching properties for {0} file(s)...' . format ( file_type ) ) for file_name in files : if file_name in absolute_file_paths and file_name != path : continue if file_name . lower ( ) [ - 3 : ] == 'zip' : file_stat_result = stat ( file_name ) file_properties = CatalogAttributes ( basename ( file_name ) , file_type , datetime . utcfromtimestamp ( file_stat_result . st_mtime ) , file_stat_result . st_size ) discovered_files . append ( { 'file_path' : file_name , 'file_properties' : file_properties , 'schema' : prospector [ 'schema' ] } ) else : logger . warning ( 'Non-zip file found in path and will be ignored: {0}' . format ( file_name ) ) logger . info ( 'Finished fetching properties for {0} files' . format ( file_type ) ) logger . info ( 'Harvested a total of {0} file(s)' . format ( len ( discovered_files ) ) ) return discovered_files
Traverse all specified prospector paths and determine uncataloged files.
Traverse all specified prospector paths and determine uncataloged files .
"""
    Traverse all specified prospector paths and determine uncataloged files.

    :param prospectors: file prospectors to harvest files
    :param logger: logger instance
    :return: dict list
    """ # List of files specified explicitly in the prospectors # If file was specified explicitly in prospector; do not index # it from other directory paths. This is done to make sure the # right file_type and schema is associated with the file. # Stat the file to fetch the file properties and store the relevant fields
None
prospectors
file prospectors to harvest files
file prospectors to harvest files
logger
None