sdap_ingest_manager/collections_ingester/yaml_file_collection_config.py
def read_yaml_collection_config(file, collection_callback):
    """
    Read the collection configuration in a yaml file, sort them by priority order
    and apply row_callback on each collection
    :param file: yaml file containing the configurations of the collections
    :param collection_callback: function which is applied to each collection, take as parameter a dictionary
    with keys [id, variable, path]
    :return:
    """
    with open(file, 'r') as f:
        collections = yaml.load(f, Loader=yaml.FullLoader)

    collections_array = []
    for (c_id, collection) in collections.items():
        collection['id'] = c_id
        for k, v in collection.items():
            if type(v) == str:
                collection[k] = v.strip()
        collections_array.append(collection)

    logger.info(f"collections are {collections_array}")
    sorted_collections = sorted(collections_array, key=lambda c: c['priority'])

    for collection in sorted_collections:
        collection_callback(collection)

    Read the collection configuration in a yaml file, sort them by priority order
    and apply row_callback on each collection
    :param file: yaml file containing the configurations of the collections
    :param collection_callback: function which is applied to each collection, take as parameter a dictionary
    with keys [id, variable, path]
    :return:
    
Read the collection configuration in a yaml file, sort them by priority order
and apply row_callback on each collection
Read the collection configuration in a yaml file sort them by priority order and apply row_callback on each collection
def read_yaml_collection_config(file, collection_callback):
    with open(file, 'r') as f:
        collections = yaml.load(f, Loader=yaml.FullLoader)
    collections_array = []
    for (c_id, collection) in collections.items():
        collection['id'] = c_id
        for k, v in collection.items():
            if type(v) == str:
                collection[k] = v.strip()
        collections_array.append(collection)
    logger.info(f"collections are {collections_array}")
    sorted_collections = sorted(collections_array, key=lambda c: c['priority'])
    for collection in sorted_collections:
        collection_callback(collection)
def read_yaml_collection_config ( file , collection_callback ) : with open ( file , 'r' ) as f : collections = yaml . load ( f , Loader = yaml . FullLoader ) collections_array = [ ] for ( c_id , collection ) in collections . items ( ) : collection [ 'id' ] = c_id for k , v in collection . items ( ) : if type ( v ) == str : collection [ k ] = v . strip ( ) collections_array . append ( collection ) logger . info ( f"collections are {collections_array}" ) sorted_collections = sorted ( collections_array , key = lambda c : c [ 'priority' ] ) for collection in sorted_collections : collection_callback ( collection )
Read the collection configuration in a yaml file, sort them by priority order
and apply row_callback on each collection
Read the collection configuration in a yaml file sort them by priority order and apply row_callback on each collection
"""
    Read the collection configuration in a yaml file, sort them by priority order
    and apply row_callback on each collection
    :param file: yaml file containing the configurations of the collections
    :param collection_callback: function which is applied to each collection, take as parameter a dictionary
    with keys [id, variable, path]
    :return:
    """
None
file
yaml file containing the configurations of the collections
yaml file containing the configurations of the collections
collection_callback
function which is applied to each collection, take as parameter a dictionary
with keys [id, variable, path]
function which is applied to each collection take as parameter a dictionary with keys [ id variable path ]