# Sometimes Jan calendar will have counts from days
# at beginning of Feb or end or previous Dec.
# Just checking that they agree w/ numbers in
# those months' calendars before dropping dupe
# dates
poldf = pollen_data(yrmths)
check_one2one(poldf, 'Date', 'Count')

poldf = poldf.drop_duplicates('Date').reset_index(drop=1)
poldf.loc[poldf.Count == -1, 'Count'] = np.nan
poldf = poldf.dropna(axis=0)
poldf.Date = pd.to_datetime(poldf.Date)

poldf = poldf.assign(
    Yr =lambda x: x.Date.dt.year,
    M  =lambda x: x.Date.dt.month,
    D  =lambda x: x.Date.dt.day,
    Doy=lambda x: x.Date.dt.dayofyear,
)

poldf['Dayi'] = (poldf.Date - poldf.Date.min()).dt.days
poldf['Prev_cnt'] = poldf.Count.shift(1).fillna(method='ffill').fillna(0)  #.interpolate()
poldf['Prev_cnt_null'] = poldf.Dayi.shift() != (poldf.Dayi - 1)

from pandas.util.testing import assert_frame_equal

No = len(poldf2)

assert_frame_equal(poldf.reset_index(drop=1)[:No], poldf2.reset_index(drop=1))

feather.write_dataframe(poldf, 'cache/pollen.fth')

poldf2 = feather.read_dataframe('cache/pollen.fth')

"""
Explanation: for yr in range(2000, 2018):
    for m in range(1, 13):
        u = url.format(year=yr, month=m)
        r = requests.get(u)
        print(yr, m, end='; ')

Pollen
End of explanation
"""


def mkds_url(sdate):
    loc = '33.7490,-84.3880'
    url_temp = 'https://api.darksky.net/forecast/{key}/{loc},{time}?exclude=flags'
    return url_temp.format(key=key, loc=loc, time=mktime(sdate))


with open('KEY.txt', 'r') as f:
    key = f.read().strip()

"""
Explanation: cs = requests.session()
u = pollen_url.format(year=2017, month=4)

cs.cache.delete_url(u)
r = requests.get(u)

!rm -r cache/joblib/util/pollen_utils/pollen_date2df/
!rm -r cache/joblib/util/pollen_utils/pollen_data/

for yr in range(2000, 2018):
    for m in range(1, 13):
        u = url.format(year=yr, month=m)
        r = requests.get(u)
        print(yr, m, end='; ')

Process monthly calendar
r = requests.get(u)
soup = BeautifulSoup(r.content, "lxml")
sel = 'div.calendar-row.calendar-row-4 > div > div > span.count > a'
soup.select_one(sel)
Darksky
End of explanation
"""


def date_rng_gen(start):
    if isinstance(start, str):
        start = dt.datetime.strptime(start, '%Y-%m-%d')
    while 1:
        # print(start)
        rng = gen_yr_rng(start, backward=True)
        # print(rng[:5])
        start = rng[-1]
        for d in rng[:-1]:
            yield d

def gen_yr_rng(start, backward=True):
    start = start
    yr = start.year
    dprev = start.replace(year=yr - 1)
    rng = pd.date_range(start=dprev, end=start)
    if backward:
        return rng[::-1]
    return rng
    d2 = dt.datetime(*start.timetuple()[:-3])

def test_date_rng_gen():
    ts = list(it.islice(date_rng_gen('2017-03-01'), 400))
    a, b = ts[0], ts[-1]
    assert (pd.date_range(b, a)[::-1] == ts).all()
    
    s = pd.Series(ts)
    assert s.value_counts(normalize=0).max() == 1
    
    
def mktime(s, hour=12):
    # tm
    try:
        d = dt.datetime.strptime(s, "%Y-%d-%m")
    except TypeError:
        d = s
#     if isinstance(s, str):
#     else:
#         d = s
        
    if hour:
        d = d.replace(hour=hour)
    f = time.mktime(d.timetuple())
    return int(f)
    
test_date_rng_gen()

"""
Explanation: Time series generator
End of explanation
"""


def camel2score(s):
    return s[0].upper() + ''.join([('_' + c.lower()) if c.isupper() else c for c in s[1:]])

def parse_data(j):
    hrs = j.pop('hourly')['data']
    # global hrdf
    js = json.dumps(hrs)
    hrdf = pd.read_json(js) #.rename(columns=camel2score)
    
    _dl = j.pop('daily')
    [[dl]] = _dl.values()
    cr = j.pop('currently')
    
#     if hrdf.shape != (24,15):
#         print(hrdf.shape)
#     assert hrdf.shape == (24,15), 'Hr shape: {}'.format(hrdf.shape)
    assert sorted(j) == ['latitude', 'longitude', 'offset', 'timezone']
    return hrdf, dl, cr, j

# hrdf, dl, cr, j = parse_data(r.json())

today = '2017-04-05'

# Pull new
for d in date_rng_gen(today):
    if d.month == d.day == 1:
        print(d.date())
    u = mkds_url(d)
    if (u in rd) or ss.cache.has_url(u):
        print('.', end='')
        continue
    r = ss.get(u)
    if r.status_code == 403:
        print('Forbidden')
        break
    if r.status_code != 200:
        print(d, r.status_code)

r = ss.get(u)

r.url

"""
Explanation: Extract data
End of explanation
"""


def sync_rdd(ss, rd, stdate='2017-03-01'):
    n = 0
    for d in date_rng_gen(stdate):
        if d.month == d.day == 1:
            print(d.date(), end=' ')
        u = mkds_url(d)
        if u in rd:
            continue

        if not ss.cache.has_url(u):
            break
        r = ss.get(u)
        assert u == r.url
        rd[r.url] = r.json()
        print('.', end='')
        n += 1
    return n
        
sync_rdd(ss, rd, stdate=today)

"""
Explanation: Sync Sqlite db w/ Redis
End of explanation
"""


def accum(stdate='2017-03-01'):
    """Roll through the dates, pull out cached
    requests, and add parsed data to list"""
    dat = []

    for i, d in enumerate(date_rng_gen(stdate)):
#         if i > 20:
#             break
        if d.month == d.day == 1:
            print(d.date())
        u = mkds_url(d)
        # if ss.cache.has_url(u):
        #     r = ss.get(u)
        #     dat.append(parse_data(r.json()))
        if u in rd:
            j = rd[u]
            parsed = parse_data(j)
            dat.append(parsed)
            # print('.', end='')
        else:
            break
    return dat
    return

"""
Explanation: Accumulate and extract data
End of explanation
"""


dat = accum(stdate='2017-04-04')

hrdfs, dls, crs, _ = zip(*dat)

dailydf = pd.read_json(json.dumps(dls)).rename(columns=camel2score)

def concat(dfs):
    all_cols = {c for df in dfs for c in df}
    col_dat = defaultdict(list)
    for cname in all_cols:
        for df in dfs:
            l = len(df)
            col_dat[cname].extend(df.get(cname, [None] * l))
    return DataFrame(col_dat)

%time hr_df = concat(hrdfs).rename(columns=camel2score)

feather.write_dataframe(dailydf, 'cache/dark_day.fth')
# feather.write_dataframe(hr_df, 'cache/dark_hr.fth')

hr_dat[:2]

"""
Explanation: %lprun -f pd.read_json accum(stdate='2017-03-01')
End of explanation
"""


def rep_with_dummies_(df, col):
    df = df.copy()
    newcs = pd.get_dummies(df[col])
    for c in newcs:
        df[c] = newcs[c]
    return df.drop(col, axis=1)

def rep_with_dummies(df, cols):
    for c in cols:
        df = rep_with_dummies_(df, c)
    return df

hr_dat2 = rep_with_dummies(hr_dat, ['Icon', 'Summary', 'Precip_type'])

ns = (~(hr_dat == hr_dat))
ns.sum()

ncols = ns.sum()[ns.sum() > 0]

cn = hr_dat.eval('Cloud_cover != Cloud_cover').astype(int)

hr_dat2.corrwith(cn).sort_values(ascending=True)

hr_dat2[:3]

"""
Explanation: Analysis of nulls
End of explanation
"""


