def consolidate_tess_fitslc(lclist,
                            normalize=True,
                            filterqualityflags=False,
                            nanfilter=None,
                            timestoignore=None,
                            headerkeys=LCHEADERKEYS,
                            datakeys=LCDATAKEYS,
                            sapkeys=LCSAPKEYS,
                            pdckeys=LCPDCKEYS,
                            topkeys=LCTOPKEYS,
                            apkeys=LCAPERTUREKEYS):
    '''This consolidates a list of LCs for a single TIC object.

    NOTE: if light curve time arrays contain nans, these and their associated
    measurements will be sorted to the end of the final combined arrays.

    Parameters
    ----------

    lclist : list of str, or str
        `lclist` is either a list of actual light curve files or a string that
        is valid for glob.glob to search for and generate a light curve list
        based on the file glob. This is useful for consolidating LC FITS files
        across different TESS sectors for a single TIC ID using a glob like
        `*<TICID>*_lc.fits`.

    normalize : bool
        If True, then the light curve's SAP_FLUX and PDCSAP_FLUX measurements
        will be normalized to 1.0 by dividing out the median flux for the
        component light curve.

    filterqualityflags : bool
        If True, will remove any measurements that have non-zero quality flags
        present. This usually indicates an issue with the instrument or
        spacecraft.

    nanfilter : {'sap','pdc','sap,pdc'} or None
        Indicates the flux measurement type(s) to apply the filtering to.

    timestoignore : list of tuples or None
        This is of the form::

            [(time1_start, time1_end), (time2_start, time2_end), ...]

        and indicates the start and end times to mask out of the final
        lcdict. Use this to remove anything that wasn't caught by the quality
        flags.

    headerkeys : list
        A list of FITS header keys that will be extracted from the FITS light
        curve file. These describe the observations. The default value for this
        is given in `LCHEADERKEYS` above.

    datakeys : list
        A list of FITS column names that correspond to the auxiliary
        measurements in the light curve. The default is `LCDATAKEYS` above.

    sapkeys : list
        A list of FITS column names that correspond to the SAP flux
        measurements in the light curve. The default is `LCSAPKEYS` above.

    pdckeys : list
        A list of FITS column names that correspond to the PDC flux
        measurements in the light curve. The default is `LCPDCKEYS` above.

    topkeys : list
        A list of FITS header keys that describe the object in the light
        curve. The default is `LCTOPKEYS` above.

    apkeys : list
        A list of FITS header keys that describe the flux measurement apertures
        used by the TESS pipeline. The default is `LCAPERTUREKEYS` above.

    Returns
    -------

    lcdict
        Returns an `lcdict` (this is useable by most astrobase functions for LC
        processing).

    '''

    # if the lclist is a string, assume that we're passing in a fileglob
    if isinstance(lclist, str):

        if sys.version_info[:2] > (3,4):

            matching = glob.glob(lclist,
                                 recursive=True)
            LOGINFO('found %s LCs: %r' % (len(matching), matching))

        else:

            lcfitsdir = os.path.dirname(lclist)
            lcfitsfile = os.path.basename(lclist)
            walker = os.walk(lcfitsdir)
            matching = []
            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              lcfitsfile)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)
                        LOGINFO(
                            'found %s in dir: %s' % (repr(foundfiles),
                                                     os.path.join(root,sdir))
                        )


        if len(matching) == 0:
            LOGERROR('could not find any TESS LC files matching glob: %s' %
                     lclist)
            return None

    # if the lclist is an actual list of LCs, then use it directly
    else:

        matching = lclist

    # get the first file
    consolidated = read_tess_fitslc(matching[0],
                                    normalize=normalize,
                                    headerkeys=LCHEADERKEYS,
                                    datakeys=LCDATAKEYS,
                                    sapkeys=LCSAPKEYS,
                                    pdckeys=LCPDCKEYS,
                                    topkeys=LCTOPKEYS,
                                    apkeys=LCAPERTUREKEYS)

    # get the rest of the files
    if len(matching) > 1:

        for lcf in matching[1:]:

            consolidated = read_tess_fitslc(lcf,
                                            appendto=consolidated,
                                            normalize=normalize,
                                            headerkeys=LCHEADERKEYS,
                                            datakeys=LCDATAKEYS,
                                            sapkeys=LCSAPKEYS,
                                            pdckeys=LCPDCKEYS,
                                            topkeys=LCTOPKEYS,
                                            apkeys=LCAPERTUREKEYS)


    # get the sort indices. we use time for the columns and sectors for the
    # bits in lcinfo and varinfo
    LOGINFO('sorting by time...')

    # NOTE: nans in time will be sorted to the end of the array
    finiteind = np.isfinite(consolidated['time'])
    if np.sum(finiteind) < consolidated['time'].size:
        LOGWARNING('some time values are nan! '
                   'measurements at these times will be '
                   'sorted to the end of the column arrays.')

    # get the time sort index
    column_sort_ind = np.argsort(consolidated['time'])

    # sort the columns by time
    for col in consolidated['columns']:
        if '.' in col:
            key, subkey = col.split('.')
            consolidated[key][subkey] = (
                consolidated[key][subkey][column_sort_ind]
            )
        else:
            consolidated[col] = consolidated[col][column_sort_ind]


    info_sort_ind = np.argsort(consolidated['lcinfo']['sector'])

    # sort the keys in lcinfo
    for key in consolidated['lcinfo']:
        consolidated['lcinfo'][key] = (
            np.array(consolidated['lcinfo'][key])[info_sort_ind].tolist()
        )

    # sort the keys in varinfo
    for key in consolidated['varinfo']:
        consolidated['varinfo'][key] = (
            np.array(consolidated['varinfo'][key])[info_sort_ind].tolist()
        )


    # filter the LC dict if requested
    # we do this at the end
    if (filterqualityflags is not False or
        nanfilter is not None or
        timestoignore is not None):
        consolidated = filter_tess_lcdict(consolidated,
                                          filterqualityflags,
                                          nanfilter=nanfilter,
                                          timestoignore=timestoignore)

    return consolidated
def consolidate_tess_fitslc(lclist,
                            normalize=True,
                            filterqualityflags=False,
                            nanfilter=None,
                            timestoignore=None,
                            headerkeys=LCHEADERKEYS,
                            datakeys=LCDATAKEYS,
                            sapkeys=LCSAPKEYS,
                            pdckeys=LCPDCKEYS,
                            topkeys=LCTOPKEYS,
                            apkeys=LCAPERTUREKEYS):
    '''This consolidates a list of LCs for a single TIC object.

    NOTE: if light curve time arrays contain nans, these and their associated
    measurements will be sorted to the end of the final combined arrays.

    Parameters
    ----------

    lclist : list of str, or str
        `lclist` is either a list of actual light curve files or a string that
        is valid for glob.glob to search for and generate a light curve list
        based on the file glob. This is useful for consolidating LC FITS files
        across different TESS sectors for a single TIC ID using a glob like
        `*<TICID>*_lc.fits`.

    normalize : bool
        If True, then the light curve's SAP_FLUX and PDCSAP_FLUX measurements
        will be normalized to 1.0 by dividing out the median flux for the
        component light curve.

    filterqualityflags : bool
        If True, will remove any measurements that have non-zero quality flags
        present. This usually indicates an issue with the instrument or
        spacecraft.

    nanfilter : {'sap','pdc','sap,pdc'} or None
        Indicates the flux measurement type(s) to apply the filtering to.

    timestoignore : list of tuples or None
        This is of the form::

            [(time1_start, time1_end), (time2_start, time2_end), ...]

        and indicates the start and end times to mask out of the final
        lcdict. Use this to remove anything that wasn't caught by the quality
        flags.

    headerkeys : list
        A list of FITS header keys that will be extracted from the FITS light
        curve file. These describe the observations. The default value for this
        is given in `LCHEADERKEYS` above.

    datakeys : list
        A list of FITS column names that correspond to the auxiliary
        measurements in the light curve. The default is `LCDATAKEYS` above.

    sapkeys : list
        A list of FITS column names that correspond to the SAP flux
        measurements in the light curve. The default is `LCSAPKEYS` above.

    pdckeys : list
        A list of FITS column names that correspond to the PDC flux
        measurements in the light curve. The default is `LCPDCKEYS` above.

    topkeys : list
        A list of FITS header keys that describe the object in the light
        curve. The default is `LCTOPKEYS` above.

    apkeys : list
        A list of FITS header keys that describe the flux measurement apertures
        used by the TESS pipeline. The default is `LCAPERTUREKEYS` above.

    Returns
    -------

    lcdict
        Returns an `lcdict` (this is useable by most astrobase functions for LC
        processing).

    '''

    # if the lclist is a string, assume that we're passing in a fileglob
    if isinstance(lclist, str):

        if sys.version_info[:2] > (3,4):

            matching = glob.glob(lclist,
                                 recursive=True)
            LOGINFO('found %s LCs: %r' % (len(matching), matching))

        else:

            lcfitsdir = os.path.dirname(lclist)
            lcfitsfile = os.path.basename(lclist)
            walker = os.walk(lcfitsdir)
            matching = []
            for root, dirs, _files in walker:
                for sdir in dirs:
                    searchpath = os.path.join(root,
                                              sdir,
                                              lcfitsfile)
                    foundfiles = glob.glob(searchpath)

                    if foundfiles:
                        matching.extend(foundfiles)
                        LOGINFO(
                            'found %s in dir: %s' % (repr(foundfiles),
                                                     os.path.join(root,sdir))
                        )


        if len(matching) == 0:
            LOGERROR('could not find any TESS LC files matching glob: %s' %
                     lclist)
            return None

    # if the lclist is an actual list of LCs, then use it directly
    else:

        matching = lclist

    # get the first file
    consolidated = read_tess_fitslc(matching[0],
                                    normalize=normalize,
                                    headerkeys=LCHEADERKEYS,
                                    datakeys=LCDATAKEYS,
                                    sapkeys=LCSAPKEYS,
                                    pdckeys=LCPDCKEYS,
                                    topkeys=LCTOPKEYS,
                                    apkeys=LCAPERTUREKEYS)

    # get the rest of the files
    if len(matching) > 1:

        for lcf in matching[1:]:

            consolidated = read_tess_fitslc(lcf,
                                            appendto=consolidated,
                                            normalize=normalize,
                                            headerkeys=LCHEADERKEYS,
                                            datakeys=LCDATAKEYS,
                                            sapkeys=LCSAPKEYS,
                                            pdckeys=LCPDCKEYS,
                                            topkeys=LCTOPKEYS,
                                            apkeys=LCAPERTUREKEYS)


    # get the sort indices. we use time for the columns and sectors for the
    # bits in lcinfo and varinfo
    LOGINFO('sorting by time...')

    # NOTE: nans in time will be sorted to the end of the array
    finiteind = np.isfinite(consolidated['time'])
    if np.sum(finiteind) < consolidated['time'].size:
        LOGWARNING('some time values are nan! '
                   'measurements at these times will be '
                   'sorted to the end of the column arrays.')

    # get the time sort index
    column_sort_ind = np.argsort(consolidated['time'])

    # sort the columns by time
    for col in consolidated['columns']:
        if '.' in col:
            key, subkey = col.split('.')
            consolidated[key][subkey] = (
                consolidated[key][subkey][column_sort_ind]
            )
        else:
            consolidated[col] = consolidated[col][column_sort_ind]


    info_sort_ind = np.argsort(consolidated['lcinfo']['sector'])

    # sort the keys in lcinfo
    for key in consolidated['lcinfo']:
        consolidated['lcinfo'][key] = (
            np.array(consolidated['lcinfo'][key])[info_sort_ind].tolist()
        )

    # sort the keys in varinfo
    for key in consolidated['varinfo']:
        consolidated['varinfo'][key] = (
            np.array(consolidated['varinfo'][key])[info_sort_ind].tolist()
        )


    # filter the LC dict if requested
    # we do this at the end
    if (filterqualityflags is not False or
        nanfilter is not None or
        timestoignore is not None):
        consolidated = filter_tess_lcdict(consolidated,
                                          filterqualityflags,
                                          nanfilter=nanfilter,
                                          timestoignore=timestoignore)

    return consolidated
def consolidate_tess_fitslc ( lclist , normalize = True , filterqualityflags = False , nanfilter = None , timestoignore = None , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS ) : # if the lclist is a string, assume that we're passing in a fileglob if isinstance ( lclist , str ) : if sys . version_info [ : 2 ] > ( 3 , 4 ) : matching = glob . glob ( lclist , recursive = True ) LOGINFO ( 'found %s LCs: %r' % ( len ( matching ) , matching ) ) else : lcfitsdir = os . path . dirname ( lclist ) lcfitsfile = os . path . basename ( lclist ) walker = os . walk ( lcfitsdir ) matching = [ ] for root , dirs , _files in walker : for sdir in dirs : searchpath = os . path . join ( root , sdir , lcfitsfile ) foundfiles = glob . glob ( searchpath ) if foundfiles : matching . extend ( foundfiles ) LOGINFO ( 'found %s in dir: %s' % ( repr ( foundfiles ) , os . path . join ( root , sdir ) ) ) if len ( matching ) == 0 : LOGERROR ( 'could not find any TESS LC files matching glob: %s' % lclist ) return None # if the lclist is an actual list of LCs, then use it directly else : matching = lclist # get the first file consolidated = read_tess_fitslc ( matching [ 0 ] , normalize = normalize , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS ) # get the rest of the files if len ( matching ) > 1 : for lcf in matching [ 1 : ] : consolidated = read_tess_fitslc ( lcf , appendto = consolidated , normalize = normalize , headerkeys = LCHEADERKEYS , datakeys = LCDATAKEYS , sapkeys = LCSAPKEYS , pdckeys = LCPDCKEYS , topkeys = LCTOPKEYS , apkeys = LCAPERTUREKEYS ) # get the sort indices. we use time for the columns and sectors for the # bits in lcinfo and varinfo LOGINFO ( 'sorting by time...' ) # NOTE: nans in time will be sorted to the end of the array finiteind = np . isfinite ( consolidated [ 'time' ] ) if np . sum ( finiteind ) < consolidated [ 'time' ] . size : LOGWARNING ( 'some time values are nan! ' 'measurements at these times will be ' 'sorted to the end of the column arrays.' ) # get the time sort index column_sort_ind = np . argsort ( consolidated [ 'time' ] ) # sort the columns by time for col in consolidated [ 'columns' ] : if '.' in col : key , subkey = col . split ( '.' ) consolidated [ key ] [ subkey ] = ( consolidated [ key ] [ subkey ] [ column_sort_ind ] ) else : consolidated [ col ] = consolidated [ col ] [ column_sort_ind ] info_sort_ind = np . argsort ( consolidated [ 'lcinfo' ] [ 'sector' ] ) # sort the keys in lcinfo for key in consolidated [ 'lcinfo' ] : consolidated [ 'lcinfo' ] [ key ] = ( np . array ( consolidated [ 'lcinfo' ] [ key ] ) [ info_sort_ind ] . tolist ( ) ) # sort the keys in varinfo for key in consolidated [ 'varinfo' ] : consolidated [ 'varinfo' ] [ key ] = ( np . array ( consolidated [ 'varinfo' ] [ key ] ) [ info_sort_ind ] . tolist ( ) ) # filter the LC dict if requested # we do this at the end if ( filterqualityflags is not False or nanfilter is not None or timestoignore is not None ) : consolidated = filter_tess_lcdict ( consolidated , filterqualityflags , nanfilter = nanfilter , timestoignore = timestoignore ) return consolidated
This consolidates a list of LCs for a single TIC object.

    NOTE: if light curve time arrays contain nans, these and their associated
    measurements will be sorted to the end of the final combined arrays.

    Parameters
    ----------

    lclist : list of str, or str
        `lclist` is either a list of actual light curve files or a string that
        is valid for glob.glob to search for and generate a light curve list
        based on the file glob. This is useful for consolidating LC FITS files
        across different TESS sectors for a single TIC ID using a glob like
        `*<TICID>*_lc.fits`.

    normalize : bool
        If True, then the light curve's SAP_FLUX and PDCSAP_FLUX measurements
        will be normalized to 1.0 by dividing out the median flux for the
        component light curve.

    filterqualityflags : bool
        If True, will remove any measurements that have non-zero quality flags
        present. This usually indicates an issue with the instrument or
        spacecraft.

    nanfilter : {'sap','pdc','sap,pdc'} or None
        Indicates the flux measurement type(s) to apply the filtering to.

    timestoignore : list of tuples or None
        This is of the form::

            [(time1_start, time1_end), (time2_start, time2_end), ...]

        and indicates the start and end times to mask out of the final
        lcdict. Use this to remove anything that wasn't caught by the quality
        flags.

    headerkeys : list
        A list of FITS header keys that will be extracted from the FITS light
        curve file. These describe the observations. The default value for this
        is given in `LCHEADERKEYS` above.

    datakeys : list
        A list of FITS column names that correspond to the auxiliary
        measurements in the light curve. The default is `LCDATAKEYS` above.

    sapkeys : list
        A list of FITS column names that correspond to the SAP flux
        measurements in the light curve. The default is `LCSAPKEYS` above.

    pdckeys : list
        A list of FITS column names that correspond to the PDC flux
        measurements in the light curve. The default is `LCPDCKEYS` above.

    topkeys : list
        A list of FITS header keys that describe the object in the light
        curve. The default is `LCTOPKEYS` above.

    apkeys : list
        A list of FITS header keys that describe the flux measurement apertures
        used by the TESS pipeline. The default is `LCAPERTUREKEYS` above.

    Returns
    -------

    lcdict
        Returns an `lcdict` (this is useable by most astrobase functions for LC
        processing).
This consolidates a list of LCs for a single TIC object .