nmstoget = ('Dune', 'AnaK', 'Odyssey', 'Gilgamesh', 'OBrother', 'Narnia', 'Catch22')
hosts_to_target = [h for h in hsd.values() if h.name in nmstoget]
assert len(hosts_to_target)==len(nmstoget)
new_targets = [hosts.NSAHost(145729), hosts.NSAHost(21709)]
hosts_to_target.extend(new_targets)

# now set to the latest base catalogs
for h in hsd.values() + hosts_to_target:
    h.fnsdss = 'SAGADropbox/base_catalogs/base_sql_nsa{0}.fits.gz'.format(h.nsaid)
    h._cached_sdss = None

# make sure the catalogs are loaded
for h in hosts_to_target:
    h.get_sdss_catalog()

hosts_to_target

# these are the already-observed objects

# we use "h" for this only as a convenience because the method adds some useful bits that aren't in the raw catalogs
spectra = h.load_and_reprocess_sdss_catalog('SAGADropbox/data/saga_spectra_clean.fits.gz')

"""
Explanation: Prepare catalogs
End of explanation
"""


for h in hosts_to_target:
    print('On host', h.name)
    cat = h.get_sdss_catalog()

    r0 = cat['r'] - cat['Ar']
    
    cutbyrawmagcut = (r0<20.75)&(cat['r']>21)
    if np.sum(cutbyrawmagcut)>0:
        print('WARNING: ', np.sum(cutbyrawmagcut), 'objects have rmags in selection but are extincted too much for r>21')
    pris = np.ones(len(cat))
    
    # remove list
    tokeep = cat['REMOVE']==-1
    pris[~tokeep] = -cat['REMOVE'][~tokeep] # sets the REMOVE objects to -their remove value
    remmskval =  np.min(pris)-1
    # remove anything in the remove list online but not in the catalog as remove
    pris[~targeting.remove_targets_with_remlist(cat, h, maskonly=True, verbose='warning')&(pris>-1)] = remmskval
    if np.sum(pris==remmskval) > 0:
        print('Removed', np.sum(pris==remmskval), 'due to online remove list. Remmsk val:', remmskval)
        
        
    photgood = ((cat['r'] < 21.) & # this is to cut the numbers a bit - more stringent mag cuts below
                (cat['fibermag_r']<23) & 
                (cat['phot_sg']=='GALAXY'))
    nearish = (cat['RHOST_KPC']<450) # again, to limit the numbers
    pris[~photgood|~nearish] = 0 
    
    # pris ~-100 are removed due to already-observed
    nospec = cat['ZQUALITY']<3
    pris[~nospec] = -100-cat['ZQUALITY'][~nospec]
    
    brighter = r0 < 20
    fainter = (r0 < 20.75)&~brighter
    goodcolors = targeting.colorcut_mask(cat, {'g-r': (None, 0.85, 2), 'r-i': (None, 0.55, 2)},deredden=True)
    okcolors = targeting.colorcut_mask(cat, {'g-r': (None, 1.2), 'r-i': (None, 0.7)}, deredden=False)
    inside = cat['RHOST_KPC']<300
    
    pris[(pris>0)&~inside&fainter&goodcolors] = np.max(pris) + 1
    pris[(pris>0)&~inside&brighter&goodcolors] = np.max(pris) + 1
    pris[(pris>0)&inside&fainter&okcolors] = np.max(pris) + 1
    pris[(pris>0)&inside&brighter&okcolors] = np.max(pris) + 1
    pris[(pris>0)&inside&fainter&goodcolors] = np.max(pris) + 1
    pris[(pris>0)&inside&brighter&goodcolors] = np.max(pris) + 1
    #everything left is in pri 1

    # this *shouldn't* be necessary, as ZQUALITY should be in the base catalog.  
    # But as a sanity check we look to see if anything in the spectral catalog is still being included
    spec_this_host = spectra[spectra['HOST_NSAID']==h.nsaid]
    spec_this_host = spec_this_host[np.in1d(spec_this_host['OBJID'], cat['OBJID'])]

    zq = cat['ZQUALITY'].copy()
    for i, zqi in zip(spec_this_host['OBJID'], spec_this_host['ZQUALITY']):
        zq[cat['OBJID']==i] = zqi
    if np.any(pris[zq>2]>=0):
        print('POSSIBLE PROBLEM: Found some objects in spectrum list that are *not* claimed ' 
              'as having spectra in the base catalogs.  Setting them to -11x:', dict(Counter(pris[pris<-110])))
        pris[zq>2] = -110 - zq[zq>2]
        
    #de-duplicate
    if len(np.unique(cat['OBJID'])) != len(cat):
        _, idxs = np.unique(cat['OBJID'], return_index=True)
        msk = np.ones_like(cat, dtype=bool)
        msk[idxs] = 0
        pris[msk] = -1000
        
        print('WARNING: some duplicate objid found.  Setting', np.sum(pris==-1000), 'dupes to pri=-1000')
    
    cat['aat_pris'] = pris
    if 'aat_pris_unjiggered' in cat.colnames:
        cat.remove_column('aat_pris_unjiggered')

    #informational
    counter = Counter(pris)
    print('Rank counts:')
    for k in reversed(sorted(counter)):
        print(int(k), ':', counter[k])

"""
Explanation: Prioritize
```
x = base catalog on Dropbox
; CORRECT FOR EXTINCTION
      r = x.r - x.extinction_r
      i = x.i - x.extinction_i
      g = x.g - x.extinction_g
; DEFINE GRI CRITERIA, ERRORS ARE SUBTRACTED IN QUADRATURE
     cgr = (g - r) - 2.sqrt(x.g_err^2 + x.r_err^2)
     cri = (r - i) - 2.sqrt(x.r_err^2 + x.i_err^2)
; SELECT TARGETS
   qtarget = where(x.rhost_kpc le 300 and $   ; INSIDE RVIR
                   x.fibermag_r le 23 and $   ; FIBERMAG CUT
                   x.remove eq -1 and $       ; SHRED LIST
                   x.phot_sg eq 'GALAXY'and $ ; GALAXIES ONLY
                   cgr le 0.85 and cri le 0.55 and $ ; GRI CUT
                   x.zquality lt 3)           ; NO SPECTRA
```
The scheme below is:
7: r&lt;300 kpc , M_r&lt;20, "good" colorcuts
6: r&lt;300 kpc , 20&lt;M_r&lt;20.75, "good" colorcuts
5: r&lt;300 kpc , M_r&lt;20, "ok" colorcuts
4: r&lt;300 kpc , 20&lt;M_r&lt;20.75, "ok" colorcuts
3: 450&gt;r&gt;300 kpc , M_r&lt;20, "good" colorcuts
2: 450&gt;r&gt;300 kpc , 20&lt;M_r&lt;20.75, " good" colorcuts
1: everything else with r&lt;21 (*not* M_r)
End of explanation
"""


for h in new_targets:
    print(h.name)
        
    cat = h.get_sdss_catalog()
    if 'aat_pris_unjiggered' in cat.colnames:
        print('Already rejiggered', h, 'skipping (but still reporting stats)...')
        
        pris = cat['aat_pris']
        print(dict(Counter(pris[pris>0])))
        print(dict(Counter(pris[pris<=0])))
        continue
    
    p = cat['PROBABILITY_CLASS1']
    if np.all(p<0):
        print('WARNING: host', h, 'does not have probs, so not re-jiggering based on ML probs')
        continue
    
    pris = np.array(cat['aat_pris_unjiggered' if 'aat_pris_unjiggered' in cat.colnames else 'aat_pris'])
    
    goodcolorbright = pris == 7
    goodcolorfaint = pris == 6
    okcolor = (pris==5) | (pris==4)
    
    
    pris[okcolor] = -0.75
    
    pris[goodcolorfaint] = 4
    pris[goodcolorbright] = 5
    pris[goodcolorfaint&(p>.001)] = 6
    pris[goodcolorbright&(p>.001)] = 7
    
    print(dict(Counter(pris[pris>0])))
    print(dict(Counter(pris[pris<=0])))
    
    cat['aat_pris_unjiggered'] = cat['aat_pris']
    cat['aat_pris'] = pris

"""
Explanation: Note: the 2 objects in OBrother with the warning have r~21, and are definitely not interesting targets
Re-jigger priorities for new targets to account for ML prob
The re-jiggered scheme is:
```
7: r<300 kpc , M_r<20, "good" colorcuts, ML>.001
6: r<300 kpc , 20<M_r<20.75, "good" colorcuts, ML>.001
5: r<300 kpc , M_r<20, "good" colorcuts, ML<.001
4: r<300 kpc , 20<M_r<20.75, "good" colorcuts, ML<.001
<3 : same as above
```
End of explanation
"""


special_objids = []

"""
Explanation: Some special objects
End of explanation
"""


possible_anak_sat = SkyCoord(353.7229, 1.2064, unit=u.deg)

spectrascs = SkyCoord(spectra['ra'], spectra['dec'], unit=u.deg)
row = spectra[np.argsort(possible_anak_sat.separation(spectrascs))[0]]
special_objids.append(row['OBJID'])
row

"""
Explanation: http://legacysurvey.org/viewer?ra=353.722358394&dec=1.20743944443
End of explanation
"""


del special_objids[special_objids.index(row['OBJID'])]

"""
Explanation: The above object was put as SAT==2, but is at very close z to AnaK...? r~=17.6 in DECALS.  Marla says this is because it's too far from the host, so not worth special-izing
End of explanation
"""


possible_anak_sat = SkyCoord(354.286, 0.211, unit=u.deg)

anakcat = hsd['AnaK'].get_sdss_catalog()
anakscs = SkyCoord(anakcat['ra'], anakcat['dec'], unit=u.deg)
seps = possible_anak_sat.separation(anakscs)
closest = np.argsort(seps)[0]
print(seps[closest].arcsec)
row = anakcat[closest]
special_objids.append(row['OBJID'])
row

"""
Explanation: Risa suggests: http://legacysurvey.org/viewer?ra=354.2826&dec=0.2110&zoom=15&layer=decals-dr2
End of explanation
"""


possible_anak_sat = SkyCoord(354.527, 0.533964, unit=u.deg)

seps = possible_anak_sat.separation(anakscs)
closest = np.argsort(seps)[0]
print(seps[closest].arcsec)
row = anakcat[closest]
special_objids.append(row['OBJID'])
row

"""
Explanation: This is not actually quite on the center-of-light in DECALS... but it might be an HII region?
Marla wants:RA = 354.527/DEC = 0.533964
End of explanation
"""


def find_risa_objs(stringfromrisa, h):
    risa_tab = Table.read(stringfromrisa, delimiter='\t', format='ascii', names=['unk', 'ra', 'dec', 'mag', 'cand'])
    risa_sc = SkyCoord(risa_tab['ra'], risa_tab['dec'], unit=u.deg)

    cat = h.get_sdss_catalog()

    catsc = SkyCoord(cat['ra'], cat['dec'], unit=u.deg)
    idx, d2d, _ = risa_sc.match_to_catalog_sky(catsc)
    assertmsg = '{} matched of {}: {}'.format(np.sum(d2d < .1*u.arcsec),len(d2d), d2d[d2d > .1*u.arcsec])
    assert np.all(d2d < .1*u.arcsec), assertmsg
    return cat['OBJID'][idx]

risa_145729="""
-1	224.56410	-1.0730193	19.7973	true
-1	224.59164	-1.1261357	17.9597	true
-1	224.58605	-1.1340797	19.9805	true
-1	224.65696	-1.1021396	17.4972	true
-1	224.54478	-1.1645862	19.4057	true
-1	224.50349	-1.1783027	17.7666	true
-1	224.61258	-1.2283750	20.1190	true
-1	224.66071	-1.2407656	20.3448	true
-1	224.58210	-1.2891033	20.1278	true
-1	224.68795	-0.82928863	18.7276	true
-1	224.46354	-0.85993860	20.6228	true
-1	224.43907	-1.3290346	20.2419	true
-1	224.27041	-1.0663297	19.6382	true
-1	224.92796	-1.0868430	19.9441	true
-1	224.95218	-1.2046019	20.2506	true
-1	224.98659	-1.0996963	19.2848	true
-1	224.95028	-1.2134533	19.1667	true
-1	224.56810	-0.71035594	19.5400	true
-1	224.56710	-0.71155324	18.3361	true
-1	224.63475	-0.76637428	20.5220	true
-1	224.79342	-0.82424335	19.5245	true
-1	224.26293	-1.3190454	19.7427	true
-1	224.34037	-1.3851494	19.5061	true
-1	224.67776	-1.3717603	18.9769	true
-1	224.30819	-0.89372642	19.5476	true
-1	224.95888	-1.0081097	19.6524	true
-1	225.01145	-1.2106150	19.6745	true
-1	224.27946	-0.80572367	19.0886	true
-1	224.44473	-0.64135326	18.2103	true
-1	224.59702	-0.60626247	19.4427	true
-1	224.98059	-1.2775413	20.7447	true
-1	224.25056	-1.3830278	20.6792	true
-1	224.03729	-1.0589027	20.7401	true
-1	224.94320	-0.87332390	19.9586	true
-1	224.12169	-1.2418469	18.6920	true
-1	225.09967	-1.2117895	19.8792	true
-1	224.28313	-0.67401930	20.1558	true
-1	224.18769	-0.79627184	19.9399	true
-1	224.23305	-0.67032897	20.2131	true
-1	225.00922	-0.80628957	20.5866	true
-1	224.32848	-1.5812675	18.2125	true
-1	224.27623	-1.5292467	18.4006	true
-1	224.70055	-1.6463751	18.5479	true
-1	225.06682	-1.2727903	20.5982	true
-1	224.89664	-1.5217602	19.0338	true
-1	225.02588	-1.4044669	20.2629	true
-1	224.98083	-1.4368200	20.4261	true
-1	225.07035	-0.95870445	19.6174	true
-1	224.14144	-0.71374995	20.3682	true
-1	224.18156	-0.65458846	19.8804	true
-1	224.03748	-0.86010033	20.2428	true
-1	224.29784	-1.5985485	19.0072	true
-1	224.30080	-1.5957333	20.6291	true
-1	224.65269	-1.6814901	20.2405	true
-1	224.18598	-1.4982058	19.6720	true
-1	225.18215	-0.98714751	20.2422	true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_145729, [h for h in hosts_to_target if h.nsaid==145729][0])

risa_obrother="""
1	335.97281	-3.4295662	17.7332	true
1	335.91314	-3.5404510	19.7043	true
-1	335.77781	-3.4459616	20.1615	true
-1	335.81490	-3.6025596	20.5123	true
1	336.04145	-3.2204987	20.5081	true
-1	336.09493	-3.4649021	19.7341	true
1	335.99401	-3.7007769	20.5931	true
1	336.12273	-3.5101925	20.4868	true
1	335.72556	-3.1674595	20.6372	true
1	335.84376	-3.0261104	20.7444	true
-1	336.23396	-3.1875586	20.7117	true
1	335.55249	-3.6052065	20.0624	true
1	335.65592	-3.6558837	20.5213	true
-1	335.57983	-3.6963397	19.3788	true
-1	336.30042	-3.4636766	19.8654	true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_obrother, hsd['OBrother']))

risa_catch22="""
-1	348.62149	4.5071708	17.7932	true
-1	348.73347	4.5865011	19.4766	true
-1	348.67493	4.6123235	20.3472	true
-1	348.72361	4.4495699	20.6729	true
-1	348.72881	4.4323062	19.4804	true
-1	348.55899	4.5900220	20.2576	true
-1	348.64485	4.4040044	17.6392	true
-1	348.59640	4.3492465	20.6181	true
-1	348.68132	4.3095517	20.6868	true
-1	348.68817	4.3020869	20.7035	true
-1	348.89822	4.4892740	18.9281	true
-1	348.43132	4.7245873	19.3515	true
-1	348.51966	4.7464873	19.3880	true
-1	348.39920	4.5666321	18.2252	true
-1	348.99115	4.5918658	20.7471	true
-1	348.31622	4.3290159	19.6409	true
-1	348.87290	4.8064919	18.9324	true
-1	348.63961	4.2011104	20.5785	true
-1	348.98746	4.3261837	18.3678	true
-1	348.40085	4.8681252	17.4557	true
-1	348.64976	4.8775596	18.4048	true
-1	348.82739	4.9078081	19.3681	true
-1	348.68761	4.9226928	17.8673	true
-1	348.85846	4.8346772	19.6157	true
"""[1:-1]
special_objids.extend(find_risa_objs(risa_catch22, hsd['Catch22']))

risa_narnia="""
"""[1:-1]
special_objids.extend(find_risa_objs(risa_narnia, hsd['Narnia']))

for soid in special_objids:
    for h in hosts_to_target:
        cat = h.get_sdss_catalog()
        if soid in cat['OBJID']:
            print('Found oid', soid, 'in', h.name, 'so setting to pri=9')
            pris = cat['aat_pris']
            if np.any(pris[np.in1d(cat['OBJID'], soid)]<1):
                print('WARNING: ', np.sum(pris[np.in1d(cat['OBJID'], soid)]<1), 'special objects in', 
                      h.name, 'are pri<0... skipping')
            cat['aat_pris'][np.in1d(cat['OBJID'], soid)&(pris>0)] = 9
            break
    else:
        print('Could not find oid', soid, 'anywhere')

"""
Explanation: That one seems good
Risa's eye-balled selections
End of explanation
"""


# commented because it should only be run once
!ls aat_targets_jun2015/*_sky.dat
#!cp aat_targets_jun2015/*_sky.dat aat_targets_jul2016/

"""
Explanation: Make the master catalogs
Sky positions
For some hosts we already have sky positions from the last run, so copy those over
End of explanation
"""


#Identify sky regions for each host and write out to separate files - 
from os.path import exists

for h in hosts_to_target:
    outfn = 'aat_targets_jul2016/' + h.name.replace(' ','_') + '_sky.dat'
    if exists(outfn):
        print(outfn, 'exists, not overwriting')
    else:
        try:
            h.get_usnob_catalog()
            usnocat = None
        except IOError:
            # this is currently hanging as the usno server is down...
            #print('Downloading USNO B1 catalog for', h.name)  
            #h.usnob_environs_query(dl=True) 
            #usnocat = None
            usnocat = False
        aat.select_sky_positions(h, nsky=100, outfn=outfn, rad=1*u.deg, usnocat=usnocat)
        aat.imagelist_fld_targets(outfn, ttype='sky', n=np.inf)
        !subl $outfn
        raw_input('Wrote ' + outfn + '. Press enter to continue onto the next one once '
                  'you remove bad entries from this file.')

"""
Explanation: For the remainder, generate and visually inspect them one at a time. Edit the file to remove any that are not good sky positions
End of explanation
"""


for h in hosts_to_target:
    cat = h.get_sdss_catalog()
    pris = cat['aat_pris']
    
    guides = aat.select_guide_stars_sdss(cat)
    calibs = aat.select_flux_stars(cat, onlyoutside=300*u.kpc)
    skyradec = 'aat_targets_jul2016/{0}_sky.dat'.format(h.name)
    aat.produce_master_fld(h, datetime.date(2016, 7, 28), cat[pris>0], pris=pris[pris>0].astype(int),
                           guidestars=guides, fluxstars=calibs,skyradec=skyradec,
                           outfn='aat_targets_jul2016/{}_master.fld'.format(h.name), 
                           randomizeorder=True, inclhost=False)

"""
Explanation: Actually generate the master catalogs
End of explanation
"""


def do_subsampling(h, finum, n_in_pri, **kwargs):
    assert h in hosts_to_target
    
    n_in_pri = dict(n_in_pri)  #copy just in case
    for i in range(9):
        if i+1 not in n_in_pri:
            n_in_pri[i+1] = np.inf

    fnbase = 'aat_targets_jul2016/' + h.name
    
    fnmaster = fnbase + '_master.fld'
    fnconfig = kwargs.pop('fnconfig', fnbase + '_{0}.fld'.format(finum))
    print('Writing', fnconfig, 'from master', fnmaster)
            
    kwargs.setdefault('nflux', 5)
    kwargs.setdefault('nguides', 30)
    kwargs.setdefault('fieldname', str(finum))
    
    if 'listorem' not in kwargs:
        listorem = []
        for i in range(1, finum):
            globpat = fnbase + '_' + str(i) + '*.lis'
            listorem.extend(glob(globpat))
        assertmsg = 'Only got ' + str(len(listorem)) +' .lis file(s):'+str(listorem)+', expected ' + str(finum-1)
        assert len(listorem) == (finum-1), assertmsg
        kwargs['listorem'] = listorem
    
    aat.subsample_from_master_fld(fnmaster, fnconfig, n_in_pri, **kwargs)
    
    return fnconfig

"""
Explanation: Observations
We sub-sample from the master lists generated above for individual configurations.
End of explanation
"""


def make_logtab(logs, skip_res=[]):
    """
    Generates a table with the AAT observing logs.  
    
    ``logs`` can be a filename or list of lines from fitsheader
    """
    
    logtab = Table()
    logtab['num'] = [1]
    logtab['ccd'] = [2]
    logtab['UTDATE'] = ['2015:06:19']
    logtab['UTSTART'] = ['08:56:19']
    logtab['OBJECT'] = ['a'*100]
    logtab['TOTALEXP'] = ['a'*20]
    logtab['RUNCMD'] = ['a'*20]
    logtab['GRATID'] = ['a'*20]
    logtab['SOURCE'] = ['plate 0 ']
    logtab['MEANRA'] = [1.1]
    logtab['MEANDEC'] = [1.1]
    logtab['CFG_FILE'] = ['Odyssey_1p0xxxxxxxxxx.sds']
    logtab = logtab[1:]

    def add_logcol(accumulated_lines):
        l = accumulated_lines[0]
        hdunum = int(l.split()[2])
        hdupath = l.split()[4][:-1]
        if hdunum != 0:
            return
        for rex in skip_res:
            if re.match(rex, hdupath):
                print('Skipping', hdupath, 'because it has', rex)
                return 

        items = {'num': int(hdupath.split('/')[-1].split('.')[0][5:]),
                 'ccd': int(hdupath.split('/')[-2][4:])}

        for l in accumulated_lines[1:]:
            if '=' in l:
                nm, data = l.split('=')
                nm = nm.strip()
                data = data.split('/')[:2]
                if len(data)> 1:
                    comment = data[1]
                data = data[0].replace("'", '').strip()

                if nm in logtab.colnames:
                    items[nm] = data
        logtab.add_row(items)

    accumulated_lines = None
    if isinstance(logs, basestring):
        with open(logs) as f:
            loglines = list(f)
    else:
        loglines = logs

    for l in loglines:
        if l.startswith('# HDU'):
            if accumulated_lines is not None:
                add_logcol(accumulated_lines)
            accumulated_lines = [l]

        elif l.strip() == '':
            continue
        elif accumulated_lines is not None:
            accumulated_lines.append(l)
    return logtab

sshtarget = 'visitor3@aatlxa'

"""
Explanation: This function parses a dump of all the image headers from 'fitsheader ccd_?/.fits'
End of explanation
"""


fnconfig = do_subsampling(hsd['Dune'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Odyssey'], 1, {1:200, 2:200, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['OBrother'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Catch22'], 1, {1:100, 2:50, 3:100})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['AnaK'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

"""
Explanation: Night 1
Generate configuration field files
End of explanation
"""


# use this to get the log data from the remote machine
rem_fitsheader_loc = '/home/visitor3/miniconda2/bin/fitsheader'
datadir = '/data_lxy/aatobs/OptDet_data/160728'
fitsheader_output = !ssh $sshtarget "$rem_fitsheader_loc $datadir/ccd_?/*.fits"

# use this to use the local file to make the log
fitsheader_output = !fitsheader aat_data/28jul2016/ccd_?/*.fits

logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)

comments_jul28 = """
* At start of night weather clear, but high humidity and possible fog
* The Dune_1 observations have the blue side centered 20 angstroms further to the red than the remaining 
  observations.  The initial setup provides a bit more overlap but after inspecting a bit we concluded that it 
  would be better to tryto make sure 3727 falls into the image at the blue side
* start of Dune_1 : seeing ~1.5-2"
* start of Odyssey_1 : seeing ~2-3"
* start of OBrother_1: seeing ~4-5"
* near end of OBrother_1: seeing ~2.5". Sudden improvement in weather, humidity fell/fog disappeared, 
* start of Catch22_1: seeing ~3"
* start of AnaK_1: seeing ~1.9"
"""[1:-1]
display.Markdown(comments_jul28)

logfn = 'aat_data/28jul2016/log_jul28'

logtab.write(logfn,format='ascii')
with open(logfn, 'a') as f:
    f.write('\nComments:\n')
    f.write(comments_jul28)

"""
Explanation: Make observing log
End of explanation
"""


fnconfig = do_subsampling(hsd['Dune'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Gilgamesh'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['OBrother'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Catch22'], 2, {1:50, 2:50, 3:200})
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Narnia'], 1, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

"""
Explanation: Night 2
Note: master catalogs changed due to Risa's custom targets, so some added pri=9 objects appear even in repeats
End of explanation
"""


# use this to use the local file to make the log
fitsheader_output = !fitsheader aat_data/29jul2016/ccd_?/*.fits

logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)

comments_jul29 = """
* start of Dune_2 : seeing ~1.3"
* start of Gilgamesh_1 : seeing ~1.6"
* start of Odyssey_1a : seeing ~1.4"
* start of OBrother_2 : seeing ~1.7"
* temperature rose dramatically near end of OBrother_2 - might lead to variable seeing
* start of Catch22_2: seeing ~2.3"
* start of Narnia_1: seeing ~1.5"
"""[1:-1]
display.Markdown(comments_jul29)

logfn = 'aat_data/29jul2016/log_jul29'

logtab.write(logfn,format='ascii')
with open(logfn, 'a') as f:
    f.write('\nComments:\n')
    f.write(comments_jul29)

"""
Explanation: Make observing log
End of explanation
"""


hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 1, {1:25, 2:25, 3:50, 4:100})
!scp $fnconfig "$sshtarget:configure/"

"""
Explanation: Nights 3+
These nights are by the team that's doing the time-trade.  They will be primarily second-half, but with a little time in the first half
End of explanation
"""


hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 2, {1:25, 2:25, 3:50, 4:100}, 
                          listorem=['aat_targets_jul2016/NSA145729_1_p0.lis'], 
                          fnconfig='aat_targets_jul2016/NSA145729_2_after1p0.fld')
!scp $fnconfig "$sshtarget:configure/"

hs = [h for h in hosts_to_target if h.nsaid==145729]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 2, {1:25, 2:25, 3:50, 4:100}, 
                          listorem=['aat_targets_jul2016/NSA145729_1_p1.lis'], 
                          fnconfig='aat_targets_jul2016/NSA145729_2_after1p1.fld')
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Catch22'], 3, {1:100, 2:100, 3:300)
!scp $fnconfig "$sshtarget:configure/"

fnconfig = do_subsampling(hsd['Narnia'], 2, {1:100, 2:100, 3:300})
!scp $fnconfig "$sshtarget:configure/"

hs = [h for h in hosts_to_target if h.nsaid==21709]
assert len(hs) == 1
fnconfig = do_subsampling(hs[0], 1, {1:25, 2:25, 3:50, 4:100})
!scp $fnconfig "$sshtarget:configure/"

"""
Explanation: NSA 145729 is being treated a bit oddly because there might be two back-to-back-ish observations.  So we're making one config in advance for both the p0 and p1 configurations, and making fld's for config 2 assuming either of those.
End of explanation
"""


# fitsheader run on the output downloaded to another computer
with open('/Users/erik/tmp/fitsheader_jul30') as f:
    fitsheader_output = list(f)

logtab = make_logtab(fitsheader_output, skip_res=['.*temp.*'])
logtab[logtab['ccd']==1].show_in_notebook(display_length=20)

logfn = 'aat_data/30jul2016/log_jul30'
logtab.write(logfn,format='ascii')

"""
Explanation: Logs for observations
End of explanation
"""


!scp "$sshtarget:configure/*.sds" aat_targets_jul2016/

# *_* because otherwise the collision matrix list comes through too
!scp "$sshtarget:configure/*_*.lis" aat_targets_jul2016/

"""
Explanation: Utilities for during the night
Pull in as-generated files from remote
Note that this assumes the .lis files were generated after the configuration was made.  If need be these can be re-created from the .sds files by starting up configure and exporting the allocated fibers list.
End of explanation
"""


listab, lisscs, lisheader = aat.load_lis_file('aat_targets_jul2016/Narnia_1_p1.lis')
guidemsk = listab['codes']=='F'
names = ['{0[ids]}_f#={0[fibnums]}'.format(row) for row in listab[guidemsk]]
print(targeting.sampled_imagelist(lisscs[guidemsk], None, names=names))

"""
Explanation: Inspect guider fields as needed
End of explanation
"""


import astroplan

"""
Explanation: Planning
End of explanation
"""


#if online
ufo = urllib2.urlopen('https://docs.google.com/spreadsheet/ccc?key=1b3k2eyFjHFDtmHce1xi6JKuj3ATOWYduTBFftx5oPp8&output=csv')
hosttab = QTable.read(ufo.read(), format='csv')
ufo.close()

#if offline
#hosttab = Table.read('SAGADropbox/hosts/host_catalog_flag0.csv')

hostscs = SkyCoord(u.Quantity(hosttab['RA'], u.deg), 
                   u.Quantity(hosttab['Dec'], u.deg), 
                   distance=u.Quantity(hosttab['distance'], u.Mpc))

allspec = Table.read('/Users/erik/Dropbox/SAGA/data/allspectaken_v5.fits.gz')

sagaobsed = allspec[~((allspec['TELNAME']=='NSA')|
                      (allspec['TELNAME']=='SDSS')|
                      (allspec['TELNAME']=='GAMA'))]
sagaobsed_nsaids = np.unique(sagaobsed['HOST_NSAID'])

#UTC time from 8:35-19:35 is AAT 18 deg window
nighttimes = Time('2016-7-28 8:45:00') + np.arange(12)*u.hour
aao = EarthLocation(lon='149:3:57.9', lat='-31:16:37.3')
aao_frame = AltAz(obstime=nighttimes, location=aao)

sunaao = get_sun(nighttimes).transform_to(aao_frame)
np.max(sunaao.alt.value)

seczs = []
for sc in hostscs:
    az = sc.transform_to(aao_frame)
    seczs.append(az.secz)
seczs = np.array(seczs)

hrsvis = np.sum((1<=seczs)&(seczs<1.75),axis=1)
visenough = hrsvis>2

aaoobs = astroplan.Observer(aao, 'Australia/NSW')
midnight = aaoobs.midnight(Time('2016-7-28'))

"""
Explanation: Find possible targets
End of explanation
"""


up_times = {}

hoststoshow = hosttab[visenough]

hosts_to_target_nsaids = []
for h in hosts_to_target:
    hosts_to_target_nsaids.append(h.nsaid)
    if h.nsaid not in hoststoshow['NSAID']:
        print('adding', h)
        hoststoshow.add_row(None)
        hoststoshow['NSAID'][-1] = h.nsaid
        hoststoshow['RA'][-1] = h.ra
        hoststoshow['Dec'][-1] = h.dec
        

with open('aat_targets_jul2016/aattargs_iobserve.dat', 'w') as f:
    for host in hoststoshow:
        already_obs = host['NSAID'] in sagaobsed_nsaids
        
        name = 'NSA'+str(host['NSAID'])
        for nm, val in hsd.items():
            if val.nsaid == host['NSAID']:
                name = nm
                if nm.startswith('NSA'):
                    name = name+'_obsed'
                break
        
        f.write(name.replace(' ','_'))
        if already_obs:
            f.write('-observed')
        f.write(' ')
        f.write(str(host['RA']) + ' ')
        f.write(str(host['Dec']) + '\n')
        
        targ = astroplan.FixedTarget(SkyCoord(host['RA'], host['Dec'], unit=u.deg), name)
        
        tpl = (name, host['NSAID'], host['RA'], host['Dec'])
        transit = aaoobs.target_meridian_transit_time(midnight, targ)
        up_times[transit.jd] = tpl
        
        
        timestoplot = transit + np.linspace(-6, 6, 100)*u.hour
        taa = aaoobs.altaz(timestoplot, targ)
        msk = taa.secz >=1
        
        color = 'g' if already_obs else 'k'
        color = 'r' if host['NSAID'] in hosts_to_target_nsaids else color
        
        plt.plot(timestoplot.plot_date[msk], taa.secz[msk], c=color)
        plt.text(transit.plot_date, aaoobs.altaz(transit, targ).secz, name, ha='center', color=color)

t0 = aaoobs.sun_rise_time(midnight, 'previous')
t1 = aaoobs.sun_set_time(midnight, 'previous')
t2 = aaoobs.twilight_evening_civil(midnight, 'previous')
t3 = aaoobs.twilight_evening_nautical(midnight, 'previous')
t4 = aaoobs.twilight_evening_astronomical(midnight, 'previous')
t5 = aaoobs.twilight_morning_astronomical(midnight, 'next')
t6 = aaoobs.twilight_morning_nautical(midnight, 'next')
t7 = aaoobs.twilight_morning_civil(midnight, 'next')
t8 = aaoobs.sun_rise_time(midnight, 'next')
t9 = aaoobs.sun_set_time(midnight, 'next')

plt.fill_between([t0.plot_date,t1.plot_date],1,3, lw=0, facecolor='y', alpha=.9)
plt.fill_between([t1.plot_date,t2.plot_date],1,3, lw=0, facecolor='y', alpha=.75)
plt.fill_between([t2.plot_date,t3.plot_date],1,3, lw=0, facecolor='y', alpha=.5)
plt.fill_between([t3.plot_date,t4.plot_date],1,3, lw=0, facecolor='y', alpha=.25)
plt.fill_between([t5.plot_date,t6.plot_date],1,3, lw=0, facecolor='y', alpha=.25)
plt.fill_between([t6.plot_date,t7.plot_date],1,3, lw=0, facecolor='y', alpha=.5)
plt.fill_between([t7.plot_date,t8.plot_date],1,3, lw=0, facecolor='y', alpha=.75)
plt.fill_between([t8.plot_date,t9.plot_date],1,3, lw=0, facecolor='y', alpha=.9)

plt.axvline(midnight.plot_date, ls=':', c='k')

plt.gca().xaxis_date(aaoobs.timezone)
plt.xlim(t1.plot_date-.05, t8.plot_date+.05)
plt.axhline(1/np.cos(65*u.deg), c='k', ls='--')  #~AAT limit
plt.ylim(2.5,1)
plt.tight_layout()

#NSA urls
for jd in sorted(up_times):
    name, nsaid, ra, dec = up_times[jd]
    print(name, 'http://www.nsatlas.org/getAtlas.html?search=nsaid&nsaID={}&submit_form=Submit'.format(nsaid))

# DECALS URLs
for jd in sorted(up_times):
    name, nsaid, ra, dec = up_times[jd]
    print(name, 'http://legacysurvey.org/viewer?ra={}&dec={}&zoom=8'.format(ra, dec))

"""
Explanation: Airmass chart
End of explanation
"""


sagaobsed[sagaobsed['HOST_NSAID']==150307].show_in_notebook()

"""
Explanation: Notes:
* NSA165082: in DECALS DR2, only z-band, looks like a group, elliptical
* NSA145398: in DECALS DR2, z-band and a bit of g and r, elliptical, otherwise good
* NSA145729: in DECALS DR2, z-band and some r, OK but somewhat near SDSS edge?
* NSA145879: in DECALS DR2, z-band and some g and r, elliptical, otherwise good
* NSA166141: in DECALS DR2, z-band and some g and r, otherwise good
* NSA149977: in DECALS DR2, z-band and some g and r, otherwise good
* NSA150578: in DECALS DR2, only z-band, otherwise good
* NSA153017: in DECALS DR2, only z-band, otherwise good
* Bandamanna: in DECALS DR2, only z-band, otherwise good
* NSA127226: otherwise good
* NSA129237: in DECALS DR2, only z-band, otherwise good
* NSA129387: otherwise good
* NSA130133: elliptical? otherwise good
* NSA130625: S0? otherwise good
* NSA131531: near SDSS edge, otherwise good
End of explanation
"""


for h in hosts_to_target:
    fnbase = 'aat_targets_jul2016/test_runs/' + h.name

    finum = 1
    fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
    fnconfig = fnbase + '_test_{0}.fld'.format(finum)
    print('Writing', fnconfig, 'from master', fnmaster)

    listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
    aat.subsample_from_master_fld(fnmaster, fnconfig,
                                  {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                                  nflux=5, nguides=30,
                                  fieldname=str(finum), listorem=listorem)
    print('')

for h in hosts_to_target:
    fnbase = 'aat_targets_jul2016/test_runs/' + h.name

    finum = 2
    fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
    fnconfig = fnbase + '_test_{0}.fld'.format(finum)
    print('Writing', fnconfig, 'from master', fnmaster)

    listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
    aat.subsample_from_master_fld(fnmaster, fnconfig,
                                  {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                                  nflux=5, nguides=30,
                                  fieldname=str(finum), listorem=listorem)
    print('')

for h in hosts_to_target:
    fnbase = 'aat_targets_jul2016/test_runs/' + h.name

    finum = 3
    fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
    fnconfig = fnbase + '_test_{0}.fld'.format(finum)
    print('Writing', fnconfig, 'from master', fnmaster)

    listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
    aat.subsample_from_master_fld(fnmaster, fnconfig,
                                  {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                                  nflux=5, nguides=30,
                                  fieldname=str(finum), listorem=listorem)
    print('')

"""
Explanation: Note: NSA150307 was called "Iliad" when it was observed on WIYN
Mock sets of configurations
End of explanation
"""


h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name

finum = 2
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)

listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
                              {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                              nflux=5, nguides=30,
                              fieldname=str(finum), listorem=listorem)

h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name

finum = 3
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)

listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
                              {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                              nflux=5, nguides=30,
                              fieldname=str(finum), listorem=listorem)

h = hsd['Catch22']
fnbase = 'aat_targets_jul2016/test_runs/' + h.name

finum = 4
fnmaster = fnbase.replace('test_runs/', '') + '_master.fld'
fnconfig = fnbase + '_test_{0}.fld'.format(finum)
print('Writing', fnconfig, 'from master', fnmaster)

listorem = [fnbase + '_test_' + str(i) + '.lis' for i in range(1, finum)]
aat.subsample_from_master_fld(fnmaster, fnconfig,
                              {1:30, 2:30, 3:30, 4:np.inf, 5:np.inf, 6:np.inf, 7:np.inf, 8:np.inf, 9:np.inf}, #8 reserved for flux
                              nflux=5, nguides=30,
                              fieldname=str(finum), listorem=listorem)

"""
Explanation: Basically there after 2 observations of  Narnia and OBrother.  AnaK and Dune 1 or maybe 2, Gilg/Ody 1.
Guess: Catch 22 requires 2 or 3
NSA 145729 and maybe 165082? 
Catch22 added later
End of explanation
"""


logtab2015 = make_logtab('aat_targets_jul2016/alljun_2015_aaomega_headers')

logtab2015[(logtab2015['ccd']==1)].show_in_notebook(display_length=20)

"""
Explanation: Log from last run
End of explanation
"""


