src/dirbs/cli/catalog.py
def cli(ctx, config, statsd, logger, run_id, conn, metadata_conn, command, metrics_root, metrics_run_root):
    """DIRBS script to catalog data files received by DIRBS Core."""
    # Store metadata
    metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                       prospectors=config.catalog_config.prospectors,
                                       perform_prevalidation=config.catalog_config.perform_prevalidation)

    harvested_files = _harvest_files(config.catalog_config.prospectors, logger)
    logger.info('Fetching files in the existing data catalog...')
    cataloged_files = _fetch_catalog_files(config)
    logger.info('Found {0} file(s) in the existing catalog'.format(len(cataloged_files)))

    uncataloged_files = [x for x in harvested_files if x['file_properties'] not in cataloged_files]
    logger.info('Discovered {0} new or modified file(s)'.format(len(uncataloged_files)))

    if len(uncataloged_files) > 0:
        logger.info('Determining catalog attributes for the discovered files...')
        uncataloged_files = _populate_file_properties(config, uncataloged_files, run_id,
                                                      config.catalog_config.perform_prevalidation, logger)
        logger.info('Finished determining catalog attributes for the discovered files')
        logger.info('Updating data catalog with new or modified files...')
        _update_catalog(uncataloged_files, config)
        logger.info('Finished updating data catalog')
    else:
        logger.info('Data catalog is already up-to-date!')
DIRBS script to catalog data files received by DIRBS Core.
DIRBS script to catalog data files received by DIRBS Core.
DIRBS script to catalog data files received by DIRBS Core .
def cli(ctx, config, statsd, logger, run_id, conn, metadata_conn, command, metrics_root, metrics_run_root):
    metadata.add_optional_job_metadata(metadata_conn, command, run_id,
                                       prospectors=config.catalog_config.prospectors,
                                       perform_prevalidation=config.catalog_config.perform_prevalidation)
    harvested_files = _harvest_files(config.catalog_config.prospectors, logger)
    logger.info('Fetching files in the existing data catalog...')
    cataloged_files = _fetch_catalog_files(config)
    logger.info('Found {0} file(s) in the existing catalog'.format(len(cataloged_files)))
    uncataloged_files = [x for x in harvested_files if x['file_properties'] not in cataloged_files]
    logger.info('Discovered {0} new or modified file(s)'.format(len(uncataloged_files)))
    if len(uncataloged_files) > 0:
        logger.info('Determining catalog attributes for the discovered files...')
        uncataloged_files = _populate_file_properties(config, uncataloged_files, run_id,
                                                      config.catalog_config.perform_prevalidation, logger)
        logger.info('Finished determining catalog attributes for the discovered files')
        logger.info('Updating data catalog with new or modified files...')
        _update_catalog(uncataloged_files, config)
        logger.info('Finished updating data catalog')
    else:
        logger.info('Data catalog is already up-to-date!')
def cli ( ctx , config , statsd , logger , run_id , conn , metadata_conn , command , metrics_root , metrics_run_root ) : metadata . add_optional_job_metadata ( metadata_conn , command , run_id , prospectors = config . catalog_config . prospectors , perform_prevalidation = config . catalog_config . perform_prevalidation ) harvested_files = _harvest_files ( config . catalog_config . prospectors , logger ) logger . info ( 'Fetching files in the existing data catalog...' ) cataloged_files = _fetch_catalog_files ( config ) logger . info ( 'Found {0} file(s) in the existing catalog' . format ( len ( cataloged_files ) ) ) uncataloged_files = [ x for x in harvested_files if x [ 'file_properties' ] not in cataloged_files ] logger . info ( 'Discovered {0} new or modified file(s)' . format ( len ( uncataloged_files ) ) ) if len ( uncataloged_files ) > 0 : logger . info ( 'Determining catalog attributes for the discovered files...' ) uncataloged_files = _populate_file_properties ( config , uncataloged_files , run_id , config . catalog_config . perform_prevalidation , logger ) logger . info ( 'Finished determining catalog attributes for the discovered files' ) logger . info ( 'Updating data catalog with new or modified files...' ) _update_catalog ( uncataloged_files , config ) logger . info ( 'Finished updating data catalog' ) else : logger . info ( 'Data catalog is already up-to-date!' )
DIRBS script to catalog data files received by DIRBS Core.
DIRBS script to catalog data files received by DIRBS Core .
"""DIRBS script to catalog data files received by DIRBS Core.""" # Store metadata
ctx
config
statsd
logger
run_id
conn
metadata_conn
command
metrics_root
metrics_run_root