GBT_RFI_pipeline/process_new_RFI_files.py
def find_parameters_to_process_file(RFI_files_to_be_processed: list,path_to_current_RFI_files):
    """
    param: RFI_files_to_be_processed: List of all RFI files that need to be processed by the GBTIDL processing script
    param: path_to_current_RFI_files: String containing the path to all current RFI files, in which the files to be processed are contained
    return: data_to_process; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
    """
    data_to_process = []
    
    for file_to_be_processed in RFI_files_to_be_processed:
        try:
            _, line_to_start_reader = read_header(file_to_be_processed, path_to_current_RFI_files)
        except FileNotFoundError:
            print("file not found. Skipping.")
            continue

        # Read the csv-non header portion in
        with open(path_to_current_RFI_files+file_to_be_processed+"/"+file_to_be_processed+".raw.vegas/"+file_to_be_processed+".raw.vegas.index") as f:
            data = pandas.read_csv(f,delimiter='\s+|\t+|\s+\t+|\t+\s+',skiprows=line_to_start_reader,engine='python')
        # If the source is unknown, this is a bad scan. Currently, the processing IDL code we are feeding this into cannot handle files with even one bad scan. 
        # This processing IDL code is in the works to be replaced with one that is more robust. 
        if "Unknown" in data["SOURCE"]:
            print("Unknown source, skipping.")
            continue
        max_scan_number = max(data["SCAN"]) #Scans are 1-indexed
        min_scan_number = min(data["SCAN"]) 
        number_of_feeds = max(data["FDNUM"])+ 1 #FDNUM is zero-indexed
        number_of_IFs = max(data["IFNUM"]) + 1 #IFNUM is zero-indexed

        # Getting the first part of the filename (i.e. TRFI_122001_PF1) and taking the last component (PF1) that actually names the receiver
        receiver_name = file_to_be_processed.split("_")[-1]

        # The receiver name is not consistent in it's naming scheme. This changes the receiver name to the GBT standardized frontend naming scheme
        verified_receiver_name = GBT_receiver_specs.FrontendVerification(receiver_name)
        if verified_receiver_name == 'Unknown':
            print("Unknown Receiver. Skipping.")
            continue
        scanlist = list(range(min_scan_number,max_scan_number))
        
        # Figure out the ymax value using the info from the dictionary above
        ymax = ymax_determiner[verified_receiver_name]


       

        # Creating a dictionary for each file, append to a list of dictionaries 
        oneline_data_to_be_processed = {
            "filename": file_to_be_processed, 
            "frontend": verified_receiver_name,
            "list_of_scans": scanlist,
            "number_of_feeds":number_of_feeds,
            "number_of_IFs":number_of_IFs,
            "ymax":ymax
        }

        # Appending oneline data to be processed to create a list of lists for the file info
        data_to_process.append(oneline_data_to_be_processed)

        print(str(file_to_be_processed)+" parameter data gleaned")
    return(data_to_process)

    param: RFI_files_to_be_processed: List of all RFI files that need to be processed by the GBTIDL processing script
    param: path_to_current_RFI_files: String containing the path to all current RFI files, in which the files to be processed are contained
    return: data_to_process; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
    
List of all RFI files that need to be processed by the GBTIDL processing script
param: path_to_current_RFI_files: String containing the path to all current RFI files, in which the files to be processed are contained
return: data_to_process; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
List of all RFI files that need to be processed by the GBTIDL processing script param : path_to_current_RFI_files : String containing the path to all current RFI files in which the files to be processed are contained return : data_to_process ; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
def find_parameters_to_process_file(RFI_files_to_be_processed: list,path_to_current_RFI_files):
    data_to_process = []
    for file_to_be_processed in RFI_files_to_be_processed:
        try:
            _, line_to_start_reader = read_header(file_to_be_processed, path_to_current_RFI_files)
        except FileNotFoundError:
            print("file not found. Skipping.")
            continue
        with open(path_to_current_RFI_files+file_to_be_processed+"/"+file_to_be_processed+".raw.vegas/"+file_to_be_processed+".raw.vegas.index") as f:
            data = pandas.read_csv(f,delimiter='\s+|\t+|\s+\t+|\t+\s+',skiprows=line_to_start_reader,engine='python')
        if "Unknown" in data["SOURCE"]:
            print("Unknown source, skipping.")
            continue
        max_scan_number = max(data["SCAN"]) 
        min_scan_number = min(data["SCAN"]) 
        number_of_feeds = max(data["FDNUM"])+ 1 
        number_of_IFs = max(data["IFNUM"]) + 1 
        receiver_name = file_to_be_processed.split("_")[-1]
        verified_receiver_name = GBT_receiver_specs.FrontendVerification(receiver_name)
        if verified_receiver_name == 'Unknown':
            print("Unknown Receiver. Skipping.")
            continue
        scanlist = list(range(min_scan_number,max_scan_number))
        ymax = ymax_determiner[verified_receiver_name]
        oneline_data_to_be_processed = {
            "filename": file_to_be_processed, 
            "frontend": verified_receiver_name,
            "list_of_scans": scanlist,
            "number_of_feeds":number_of_feeds,
            "number_of_IFs":number_of_IFs,
            "ymax":ymax
        }
        data_to_process.append(oneline_data_to_be_processed)
        print(str(file_to_be_processed)+" parameter data gleaned")
    return(data_to_process)
def find_parameters_to_process_file ( RFI_files_to_be_processed : list , path_to_current_RFI_files ) : data_to_process = [ ] for file_to_be_processed in RFI_files_to_be_processed : try : _ , line_to_start_reader = read_header ( file_to_be_processed , path_to_current_RFI_files ) except FileNotFoundError : print ( "file not found. Skipping." ) continue with open ( path_to_current_RFI_files + file_to_be_processed + "/" + file_to_be_processed + ".raw.vegas/" + file_to_be_processed + ".raw.vegas.index" ) as f : data = pandas . read_csv ( f , delimiter = '\s+|\t+|\s+\t+|\t+\s+' , skiprows = line_to_start_reader , engine = 'python' ) if "Unknown" in data [ "SOURCE" ] : print ( "Unknown source, skipping." ) continue max_scan_number = max ( data [ "SCAN" ] ) min_scan_number = min ( data [ "SCAN" ] ) number_of_feeds = max ( data [ "FDNUM" ] ) + 1 number_of_IFs = max ( data [ "IFNUM" ] ) + 1 receiver_name = file_to_be_processed . split ( "_" ) [ - 1 ] verified_receiver_name = GBT_receiver_specs . FrontendVerification ( receiver_name ) if verified_receiver_name == 'Unknown' : print ( "Unknown Receiver. Skipping." ) continue scanlist = list ( range ( min_scan_number , max_scan_number ) ) ymax = ymax_determiner [ verified_receiver_name ] oneline_data_to_be_processed = { "filename" : file_to_be_processed , "frontend" : verified_receiver_name , "list_of_scans" : scanlist , "number_of_feeds" : number_of_feeds , "number_of_IFs" : number_of_IFs , "ymax" : ymax } data_to_process . append ( oneline_data_to_be_processed ) print ( str ( file_to_be_processed ) + " parameter data gleaned" ) return ( data_to_process )
param: RFI_files_to_be_processed: List of all RFI files that need to be processed by the GBTIDL processing script
param: path_to_current_RFI_files: String containing the path to all current RFI files, in which the files to be processed are contained
return: data_to_process; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
param : RFI_files_to_be_processed : List of all RFI files that need to be processed by the GBTIDL processing script param : path_to_current_RFI_files : String containing the path to all current RFI files in which the files to be processed are contained return : data_to_process ; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
"""
    param: RFI_files_to_be_processed: List of all RFI files that need to be processed by the GBTIDL processing script
    param: path_to_current_RFI_files: String containing the path to all current RFI files, in which the files to be processed are contained
    return: data_to_process; which is a list of lists containing each file with the information needed to run the GBTIDL processing script
    """ # Read the csv-non header portion in # If the source is unknown, this is a bad scan. Currently, the processing IDL code we are feeding this into cannot handle files with even one bad scan.  # This processing IDL code is in the works to be replaced with one that is more robust.  #Scans are 1-indexed #FDNUM is zero-indexed #IFNUM is zero-indexed # Getting the first part of the filename (i.e. TRFI_122001_PF1) and taking the last component (PF1) that actually names the receiver # The receiver name is not consistent in it's naming scheme. This changes the receiver name to the GBT standardized frontend naming scheme # Figure out the ymax value using the info from the dictionary above # Creating a dictionary for each file, append to a list of dictionaries  # Appending oneline data to be processed to create a list of lists for the file info
RFI_files_to_be_processed
list
path_to_current_RFI_files