Lv3_incoming.py
def filtering(eventfile,outfile,maskdet,eventflags,rm_artifacts):
    """
    Function that will filter out bad detectors and impose eventflag restrictions.
    Will expect to add to this function over time...

    eventfile - path to the event file. Will extract ObsID from this for the NICER files.
    outfile - output file from the filtering
    maskdet - list of DET_IDs to mask
    eventflags - NICER event flags to filter out (e.g., in the format '(EVENT_FLAGS==bx1x000)')
    rm_artifacts - a list of [start,end,start,end] times (specified in intervals) to REMOVE!
    Artifacts are defined as whatever you want to remove. Bursts are considered artifacts in the sense
    that they are not part of the persistent emission.
    """
    outfile_parent_folder = str(pathlib.Path(outfile).parent)

    evfilt_expr = eventflags
    for i in range(len(maskdet)):
        evfilt_expr += '.and.(DET_ID!='+str(maskdet[i]) + ')'

    if len(rm_artifacts) != 0:
        intfile = outfile_parent_folder + '/intermediate_filt.evt'
        subprocess.run(['ftcopy',eventfile+'['+evfilt_expr+']',intfile,'clobber=yes','history=yes'])

        gtis = list(fits.open(intfile)[2].data)
        rm_artifacts = fits.open(intfile)[2].data['START'][0]+np.array(open( str(pathlib.Path(eventfile).parent)+'/rm_artifacts.txt','r').read().split('\n')[:-1],dtype=np.float64)

        ### To identify GTIs that are not part of the artifacts (or bursts)
        gtis_remove = []
        print('Removing GTIs...')
        for i in tqdm(range(len(gtis))):
            for j in range(0,len(rm_artifacts)-1,2):
                if (gtis[i][0] >= rm_artifacts[j]) and (gtis[i][1] <= rm_artifacts[j+1]):
                    gtis_remove.append(gtis[i])

        new_gtis = []
        print('Getting new GTIs...')
        for i in tqdm(range(len(gtis))):
            if gtis[i] not in np.array(gtis_remove):
                new_gtis.append(gtis[i])
        ###

        gtitxt = open(outfile_parent_folder + '/filtered_gtis.txt','w')
        for i in range(len(new_gtis)):
            gtitxt.write(str(new_gtis[i][0]) + ' ' + str(new_gtis[i][1]) + '\n')
        gtitxt.close()

        gti_col_file = Lv0_dirs.NICER_DATADIR + 'gti_columns.txt'
        gti_header_file = Lv0_dirs.NICER_DATADIR + 'gti_header.txt'
        ftcreate_cmd = ['ftcreate',gti_col_file,outfile_parent_folder+'/filtered_gtis.txt',outfile_parent_folder+'/final_filtered.gti','headfile='+gti_header_file,'extname="GTI"','clobber=yes']
        subprocess.run(ftcreate_cmd)

        subprocess.run(['niextract-events',intfile,outfile,'timefile='+outfile_parent_folder+'/final_filtered.gti'])

    else:
        subprocess.run(['ftcopy',eventfile+'['+evfilt_expr+']',outfile,'clobber=yes','history=yes'])

    Function that will filter out bad detectors and impose eventflag restrictions.
    Will expect to add to this function over time...

    eventfile - path to the event file. Will extract ObsID from this for the NICER files.
    outfile - output file from the filtering
    maskdet - list of DET_IDs to mask
    eventflags - NICER event flags to filter out (e.g., in the format '(EVENT_FLAGS==bx1x000)')
    rm_artifacts - a list of [start,end,start,end] times (specified in intervals) to REMOVE!
    Artifacts are defined as whatever you want to remove. Bursts are considered artifacts in the sense
    that they are not part of the persistent emission.
    
Function that will filter out bad detectors and impose eventflag restrictions.
Will expect to add to this function over time

path to the event file. Will extract ObsID from this for the NICER files.
outfile - output file from the filtering
maskdet - list of DET_IDs to mask
eventflags - NICER event flags to filter out ')
rm_artifacts - a list of [start,end,start,end] times (specified in intervals) to REMOVE.
Artifacts are defined as whatever you want to remove. Bursts are considered artifacts in the sense
that they are not part of the persistent emission.
Function that will filter out bad detectors and impose eventflag restrictions . Will expect to add to this function over time path to the event file . Will extract ObsID from this for the NICER files . outfile - output file from the filtering maskdet - list of DET_IDs to mask eventflags - NICER event flags to filter out ' ) rm_artifacts - a list of [ start end start end ] times ( specified in intervals ) to REMOVE . Artifacts are defined as whatever you want to remove . Bursts are considered artifacts in the sense that they are not part of the persistent emission .
def filtering(eventfile,outfile,maskdet,eventflags,rm_artifacts):
    outfile_parent_folder = str(pathlib.Path(outfile).parent)
    evfilt_expr = eventflags
    for i in range(len(maskdet)):
        evfilt_expr += '.and.(DET_ID!='+str(maskdet[i]) + ')'
    if len(rm_artifacts) != 0:
        intfile = outfile_parent_folder + '/intermediate_filt.evt'
        subprocess.run(['ftcopy',eventfile+'['+evfilt_expr+']',intfile,'clobber=yes','history=yes'])
        gtis = list(fits.open(intfile)[2].data)
        rm_artifacts = fits.open(intfile)[2].data['START'][0]+np.array(open( str(pathlib.Path(eventfile).parent)+'/rm_artifacts.txt','r').read().split('\n')[:-1],dtype=np.float64)
        gtis_remove = []
        print('Removing GTIs...')
        for i in tqdm(range(len(gtis))):
            for j in range(0,len(rm_artifacts)-1,2):
                if (gtis[i][0] >= rm_artifacts[j]) and (gtis[i][1] <= rm_artifacts[j+1]):
                    gtis_remove.append(gtis[i])
        new_gtis = []
        print('Getting new GTIs...')
        for i in tqdm(range(len(gtis))):
            if gtis[i] not in np.array(gtis_remove):
                new_gtis.append(gtis[i])
        gtitxt = open(outfile_parent_folder + '/filtered_gtis.txt','w')
        for i in range(len(new_gtis)):
            gtitxt.write(str(new_gtis[i][0]) + ' ' + str(new_gtis[i][1]) + '\n')
        gtitxt.close()
        gti_col_file = Lv0_dirs.NICER_DATADIR + 'gti_columns.txt'
        gti_header_file = Lv0_dirs.NICER_DATADIR + 'gti_header.txt'
        ftcreate_cmd = ['ftcreate',gti_col_file,outfile_parent_folder+'/filtered_gtis.txt',outfile_parent_folder+'/final_filtered.gti','headfile='+gti_header_file,'extname="GTI"','clobber=yes']
        subprocess.run(ftcreate_cmd)
        subprocess.run(['niextract-events',intfile,outfile,'timefile='+outfile_parent_folder+'/final_filtered.gti'])
    else:
        subprocess.run(['ftcopy',eventfile+'['+evfilt_expr+']',outfile,'clobber=yes','history=yes'])
def filtering ( eventfile , outfile , maskdet , eventflags , rm_artifacts ) : outfile_parent_folder = str ( pathlib . Path ( outfile ) . parent ) evfilt_expr = eventflags for i in range ( len ( maskdet ) ) : evfilt_expr += '.and.(DET_ID!=' + str ( maskdet [ i ] ) + ')' if len ( rm_artifacts ) != 0 : intfile = outfile_parent_folder + '/intermediate_filt.evt' subprocess . run ( [ 'ftcopy' , eventfile + '[' + evfilt_expr + ']' , intfile , 'clobber=yes' , 'history=yes' ] ) gtis = list ( fits . open ( intfile ) [ 2 ] . data ) rm_artifacts = fits . open ( intfile ) [ 2 ] . data [ 'START' ] [ 0 ] + np . array ( open ( str ( pathlib . Path ( eventfile ) . parent ) + '/rm_artifacts.txt' , 'r' ) . read ( ) . split ( '\n' ) [ : - 1 ] , dtype = np . float64 ) gtis_remove = [ ] print ( 'Removing GTIs...' ) for i in tqdm ( range ( len ( gtis ) ) ) : for j in range ( 0 , len ( rm_artifacts ) - 1 , 2 ) : if ( gtis [ i ] [ 0 ] >= rm_artifacts [ j ] ) and ( gtis [ i ] [ 1 ] <= rm_artifacts [ j + 1 ] ) : gtis_remove . append ( gtis [ i ] ) new_gtis = [ ] print ( 'Getting new GTIs...' ) for i in tqdm ( range ( len ( gtis ) ) ) : if gtis [ i ] not in np . array ( gtis_remove ) : new_gtis . append ( gtis [ i ] ) gtitxt = open ( outfile_parent_folder + '/filtered_gtis.txt' , 'w' ) for i in range ( len ( new_gtis ) ) : gtitxt . write ( str ( new_gtis [ i ] [ 0 ] ) + ' ' + str ( new_gtis [ i ] [ 1 ] ) + '\n' ) gtitxt . close ( ) gti_col_file = Lv0_dirs . NICER_DATADIR + 'gti_columns.txt' gti_header_file = Lv0_dirs . NICER_DATADIR + 'gti_header.txt' ftcreate_cmd = [ 'ftcreate' , gti_col_file , outfile_parent_folder + '/filtered_gtis.txt' , outfile_parent_folder + '/final_filtered.gti' , 'headfile=' + gti_header_file , 'extname="GTI"' , 'clobber=yes' ] subprocess . run ( ftcreate_cmd ) subprocess . run ( [ 'niextract-events' , intfile , outfile , 'timefile=' + outfile_parent_folder + '/final_filtered.gti' ] ) else : subprocess . run ( [ 'ftcopy' , eventfile + '[' + evfilt_expr + ']' , outfile , 'clobber=yes' , 'history=yes' ] )
Function that will filter out bad detectors and impose eventflag restrictions.
Function that will filter out bad detectors and impose eventflag restrictions .
"""
    Function that will filter out bad detectors and impose eventflag restrictions.
    Will expect to add to this function over time...

    eventfile - path to the event file. Will extract ObsID from this for the NICER files.
    outfile - output file from the filtering
    maskdet - list of DET_IDs to mask
    eventflags - NICER event flags to filter out (e.g., in the format '(EVENT_FLAGS==bx1x000)')
    rm_artifacts - a list of [start,end,start,end] times (specified in intervals) to REMOVE!
    Artifacts are defined as whatever you want to remove. Bursts are considered artifacts in the sense
    that they are not part of the persistent emission.
    """ ### To identify GTIs that are not part of the artifacts (or bursts) ###
eventfile
outfile
maskdet
eventflags
rm_artifacts