pm_df = pd.read_hdf('pm_objid_stars.h5')

len(missing_is_pm_star)

len(np.where(missing_is_pm_star == 1)[0])

len(tmp_tbl)

len(np.unique(tmp_tbl.objid))

tmp_tbl

pm_objid = np.empty(0).astype(np.int64)
for mf in missing_files:
    tstart = time.time()
    tmp_tbl = fits.getdata(mf)
    unique_objid = np.unique(tmp_tbl.objid)
    missing_is_pm_star = np.isin(unique_objid, pm_df.objid.values)
    
    pm_objid = np.append(pm_objid, unique_objid[missing_is_pm_star])
    tend = time.time()
    print(mf, len(pm_objid), len(np.unique(pm_objid)), tend - tstart)

len(np.unique(pm_objid))

new_pm_stars = pd.DataFrame(pm_objid, columns=['objid'])
new_pm_stars.to_hdf('pm_stars_in_ps1_missing.h5', 'd1')

# add unique command because there are a few repeats
star_objid = np.unique(np.append(pm_objid, plx_objid)) 
new_stars = pd.DataFrame(star_objid, columns=['objid'])
new_stars.to_hdf('stars_in_ps1_missing.h5', 'd1')

gaia_in_ps1 = pd.read_hdf('stars_in_ps1_missing.h5')
star_objid = gaia_in_ps1.objid.values

print(len(star_objid) - len(np.unique(star_objid)))
print(len(np.unique(star_objid)))

"""
Explanation: Find the stars selected via proper motion
End of explanation
"""


rf_files = glob.glob('../update_*.csv')
N_gaia_and_ps1 = 0
for rff in rf_files:
    tstart = time.time()
    rf_df = pd.read_csv(rff)
    already_one = len(np.where(rf_df.score == 1)[0])
    gaia_star = np.isin(rf_df.objid.values, star_objid)
    gaia_and_ps1 = len(np.where(gaia_star == True)[0])
    N_gaia_and_ps1 += gaia_and_ps1
    update_rf_score = (gaia_star & (rf_df.score != 1))
    
    rf_df.loc[update_rf_score, "score"] = 1
    now_one = len(np.where(rf_df.score == 1)[0])
    rf_df.to_csv(rff.replace('update', 'gaia_update'), index=False)
    star_objid = star_objid[~np.isin(star_objid, rf_df.objid.values[gaia_star])]
    tend = time.time()
    print(rff, len(star_objid), gaia_and_ps1, 
          len(np.where(update_rf_score == 1)[0]), 
          tend-tstart)

N_gaia_and_ps1

len(star_objid)

gaia_only = pd.DataFrame(star_objid, columns=['objid'])

gaia_only['score'] = np.ones(len(star_objid)).astype(float)

gaia_only.head()

gaia_only.to_csv('../gaia_only_update.csv', index=False)

len(gaia_only)

len(np.unique(gaia_only.objid))

"""
Explanation: Read in RF classifications and replace Gaia stars with score = 1
End of explanation
"""


