sdap_ingest_manager/collections_ingester/collection_ingestion.py
def create_granule_list(file_path_pattern, dataset_ingestion_history_manager,
                        granule_list_file_path, deconstruct_nfs=False,
                        date_from=None, date_to=None,
                        forward_processing=False):
    """ Creates a granule list file from a file path pattern
        matching the granules.
        If a granules has already been ingested with same md5sum signature, it is not included in this list.
        When deconstruct_nfs is True, the paths will shown as viewed on the nfs server
        and not as they are mounted on the nfs client where the script runs (default behaviour).
    """

    file_list = get_file_list(file_path_pattern)

    logger.info("Create granule list file %s", granule_list_file_path)
    dir_path = os.path.dirname(granule_list_file_path)
    logger.info("Granule list file created in directory %s", dir_path)
    Path(dir_path).mkdir(parents=True, exist_ok=True)

    if forward_processing:
        if dataset_ingestion_history_manager:
            timestamp_from = dataset_ingestion_history_manager.get_latest_ingested_file_update()
        if dataset_ingestion_history_manager is None or timestamp_from is None:
            logger.info("No ingestion history available, forward processing ignored")
            timestamp_from = None
        timestamp_to = None
    else:
        timestamp_from = date_from.timestamp() if date_from else None
        timestamp_to = date_to.timestamp() if date_to else None

    if deconstruct_nfs:
        mount_points = nfs_mount_parse.get_nfs_mount_points()

    with open(granule_list_file_path, 'w') as file_handle:
        for file_path in file_list:
            if is_in_time_range(file_path, timestamp_from, timestamp_to):
                filename = os.path.basename(file_path)
                already_ingested = False
                if dataset_ingestion_history_manager:
                    logger.info(f"is file {file_path} already ingested ?")
                    already_ingested = dataset_ingestion_history_manager.has_valid_cache(file_path)
                if not already_ingested:
                    logger.info(f"file {filename} not ingested yet, added to the list")
                    if deconstruct_nfs:
                        file_path = nfs_mount_parse.replace_mount_point_with_service_path(file_path, mount_points)
                    file_handle.write(f'{file_path}\n')
                else:
                    logger.debug(f"file {filename} already ingested with same md5sum")
            else:
                logger.debug(f"file {file_path} has not been updated in the targeted time range")
 Creates a granule list file from a file path pattern
        matching the granules.
        If a granules has already been ingested with same md5sum signature, it is not included in this list.
        When deconstruct_nfs is True, the paths will shown as viewed on the nfs server
        and not as they are mounted on the nfs client where the script runs (default behaviour).
    
Creates a granule list file from a file path pattern
matching the granules.
If a granules has already been ingested with same md5sum signature, it is not included in this list.
When deconstruct_nfs is True, the paths will shown as viewed on the nfs server
and not as they are mounted on the nfs client where the script runs (default behaviour).
Creates a granule list file from a file path pattern matching the granules . If a granules has already been ingested with same md5sum signature it is not included in this list . When deconstruct_nfs is True the paths will shown as viewed on the nfs server and not as they are mounted on the nfs client where the script runs ( default behaviour ) .
def create_granule_list(file_path_pattern, dataset_ingestion_history_manager,
                        granule_list_file_path, deconstruct_nfs=False,
                        date_from=None, date_to=None,
                        forward_processing=False):
    file_list = get_file_list(file_path_pattern)
    logger.info("Create granule list file %s", granule_list_file_path)
    dir_path = os.path.dirname(granule_list_file_path)
    logger.info("Granule list file created in directory %s", dir_path)
    Path(dir_path).mkdir(parents=True, exist_ok=True)
    if forward_processing:
        if dataset_ingestion_history_manager:
            timestamp_from = dataset_ingestion_history_manager.get_latest_ingested_file_update()
        if dataset_ingestion_history_manager is None or timestamp_from is None:
            logger.info("No ingestion history available, forward processing ignored")
            timestamp_from = None
        timestamp_to = None
    else:
        timestamp_from = date_from.timestamp() if date_from else None
        timestamp_to = date_to.timestamp() if date_to else None
    if deconstruct_nfs:
        mount_points = nfs_mount_parse.get_nfs_mount_points()
    with open(granule_list_file_path, 'w') as file_handle:
        for file_path in file_list:
            if is_in_time_range(file_path, timestamp_from, timestamp_to):
                filename = os.path.basename(file_path)
                already_ingested = False
                if dataset_ingestion_history_manager:
                    logger.info(f"is file {file_path} already ingested ?")
                    already_ingested = dataset_ingestion_history_manager.has_valid_cache(file_path)
                if not already_ingested:
                    logger.info(f"file {filename} not ingested yet, added to the list")
                    if deconstruct_nfs:
                        file_path = nfs_mount_parse.replace_mount_point_with_service_path(file_path, mount_points)
                    file_handle.write(f'{file_path}\n')
                else:
                    logger.debug(f"file {filename} already ingested with same md5sum")
            else:
                logger.debug(f"file {file_path} has not been updated in the targeted time range")
def create_granule_list ( file_path_pattern , dataset_ingestion_history_manager , granule_list_file_path , deconstruct_nfs = False , date_from = None , date_to = None , forward_processing = False ) : file_list = get_file_list ( file_path_pattern ) logger . info ( "Create granule list file %s" , granule_list_file_path ) dir_path = os . path . dirname ( granule_list_file_path ) logger . info ( "Granule list file created in directory %s" , dir_path ) Path ( dir_path ) . mkdir ( parents = True , exist_ok = True ) if forward_processing : if dataset_ingestion_history_manager : timestamp_from = dataset_ingestion_history_manager . get_latest_ingested_file_update ( ) if dataset_ingestion_history_manager is None or timestamp_from is None : logger . info ( "No ingestion history available, forward processing ignored" ) timestamp_from = None timestamp_to = None else : timestamp_from = date_from . timestamp ( ) if date_from else None timestamp_to = date_to . timestamp ( ) if date_to else None if deconstruct_nfs : mount_points = nfs_mount_parse . get_nfs_mount_points ( ) with open ( granule_list_file_path , 'w' ) as file_handle : for file_path in file_list : if is_in_time_range ( file_path , timestamp_from , timestamp_to ) : filename = os . path . basename ( file_path ) already_ingested = False if dataset_ingestion_history_manager : logger . info ( f"is file {file_path} already ingested ?" ) already_ingested = dataset_ingestion_history_manager . has_valid_cache ( file_path ) if not already_ingested : logger . info ( f"file {filename} not ingested yet, added to the list" ) if deconstruct_nfs : file_path = nfs_mount_parse . replace_mount_point_with_service_path ( file_path , mount_points ) file_handle . write ( f'{file_path}\n' ) else : logger . debug ( f"file {filename} already ingested with same md5sum" ) else : logger . debug ( f"file {file_path} has not been updated in the targeted time range" )
Creates a granule list file from a file path pattern
matching the granules.
Creates a granule list file from a file path pattern matching the granules .
""" Creates a granule list file from a file path pattern
        matching the granules.
        If a granules has already been ingested with same md5sum signature, it is not included in this list.
        When deconstruct_nfs is True, the paths will shown as viewed on the nfs server
        and not as they are mounted on the nfs client where the script runs (default behaviour).
    """
file_path_pattern
dataset_ingestion_history_manager
granule_list_file_path
deconstruct_nfs
date_from
date_to
forward_processing