#! cat /Users/gully/.ipython/profile_default/startup/start.ipy

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%config InlineBackend.figure_format = 'retina'
%matplotlib inline

"""
Explanation: Gaia
Real data!
gully
Sept 14, 2016
Outline:

Batch download the data
Compare TGAS and GaiaSource
Explore

Import these first-- I auto import them every time!:
End of explanation
"""


import os

for i in range(16):
    fn = 'http://cdn.gea.esac.esa.int/Gaia/tgas_source/csv/TgasSource_000-000-{:03d}.csv.gz'.format(i)
    executable = 'wget '+fn
    print(executable)
    os.system(executable) ## Uncomment to actually download

#! mv Tgas* ../data

#! gzip -d ../data/Tgas*

! ls ../data/Tgas*

"""
Explanation: 1. Batch download the data
End of explanation
"""


#! wget http://cdn.gea.esac.esa.int/Gaia/gaia_source/csv/GaiaSource_000-000-000.csv.gz
#! mv GaiaSource_000-000-000.csv.gz ../data/

! ls ../data/GaiaSource*

"""
Explanation: Compare to a Gaia full catalog source (download from previous notebook or manually):
End of explanation
"""


import pandas as pd

%time t000 = pd.read_csv('../data/TgasSource_000-000-000.csv')

%time g000 = pd.read_csv('../data/GaiaSource_000-000-000.csv')

set(t000.columns) - set(g000.columns)

"""
Explanation: 2. Compare TGAS and GaiaSource files
End of explanation
"""


len(t000), len(g000)

p_i = t000.parallax == t000.parallax
tp000 = t000[p_i]
p_i.sum()

p_i = g000.parallax == g000.parallax
gp000 = g000[p_i]
p_i.sum()

sns.set_color_codes()

"""
Explanation: TGAS is just the subset with parallaxes available, while Gaia source has only Positions and Magnitudes but for billions of sources:
<blockquote class="twitter-tweet" data-lang="en"><p lang="en" dir="ltr"><a href="https://twitter.com/hashtag/GaiaDR1?src=hash">#GaiaDR1</a> details: 1billion stars w/ position+magnitude; 2million stars w/ pos+mag+parallax+proper motion; 3194 variable stars; 2152 quasars</p>&mdash; ESA Science (@esascience) <a href="https://twitter.com/esascience/status/776001201511010304">September 14, 2016</a></blockquote>
<script async src="//platform.twitter.com/widgets.js" charset="utf-8"></script>
End of explanation
"""


plt.plot(tp000.ra[0:2], tp000.dec[0:2], 'b.', label='TGAS') # Hack to get bold labels
plt.plot(gp000.ra[0:2], gp000.dec[0:2], 'r.', label='Gaia Source')
plt.plot(tp000.ra.values, tp000.dec.values, 'b.', alpha=0.1)
plt.plot(gp000.ra.values, gp000.dec.values, 'r.', alpha=0.1)
plt.legend(loc='lower left')

"""
Explanation: For a single file, TGAS covers much more area.  The file sizes are capped at 40 Mb.
End of explanation
"""


df_list = []

"""
Explanation: 3. Explore!
End of explanation
"""


for i in range(16):
    df_list.append(pd.read_csv('../data/TgasSource_000-000-{:03d}.csv'.format(i)))

tt = pd.concat(df_list, ignore_index=True)

t000.shape

tt.shape

len(tt.source_id.unique())

"""
Explanation: This takes a finite amount of RAM, but should be fine for modern laptops.
End of explanation
"""


plt.plot(tt.parallax, tt.parallax_error, '.', alpha=0.005)
plt.xscale('log')

bins = np.arange(-50, 200, 3)
sns.distplot(tt.parallax, bins=bins,kde=False)
plt.yscale('log')

sns.distplot(tt.parallax_error)

bins = np.arange(0, 160, 2)
sns.distplot(tt.astrometric_n_obs_ac, bins=bins, kde=False)
sns.distplot(tt.astrometric_n_bad_obs_ac, bins=bins, kde=False)
sns.distplot(tt.astrometric_n_good_obs_ac, bins=bins, kde=False)

sns.distplot(tt.phot_g_mean_mag)

bins = np.arange(0,40,1)
sns.distplot(tt.matched_observations, bins=bins,kde=False)

tt.iloc[0]

gi = tt.astrometric_delta_q == tt.astrometric_delta_q
bins= np.arange(0,500, 5)
sns.distplot(tt.astrometric_delta_q[gi], bins=bins, kde=False)
plt.yscale('log')

tt.phot_variable_flag.unique()

vi = tt.phot_variable_flag == 'VARIABLE'

vi.sum(), len(vi)

"""
Explanation: So 2.05+ million sources with 59 "features" or columns of metadata.
End of explanation
"""


tt[vi]

"""
Explanation: Only one variable star of all the TGAS sample, about what you'd expect.
End of explanation
"""


