L=json.loads(file('../json/L.json','r').read())
M=json.loads(file('../json/M.json','r').read())
N=json.loads(file('../json/N.json','r').read())

import requests

AP={}
for c in M:
    if c not in AP:AP[c]={}
    for i in range(len(L[c])):
        AP[c][N[c][i]]=L[c][i]

"""
Explanation: Load airports of each country
End of explanation
"""


baseurl='https://www.airportia.com/'
import requests, urllib2

def urlgetter(url):
    s = requests.Session()
    cookiesopen = s.get(url)
    cookies=str(s.cookies)
    fcookies=[[k[:k.find('=')],k[k.find('=')+1:k.find(' for ')]] for k in cookies[cookies.find('Cookie '):].split('Cookie ')[1:]]
    #push token
    opener = urllib2.build_opener()
    for k in fcookies:
        opener.addheaders.append(('Cookie', k[0]+'='+k[1]))
    #read html
    return s.get(url).content

"""
Explanation: record schedules for 2 weeks, then augment count with weekly flight numbers.
seasonal and seasonal charter will count as once per week for 3 months, so 12/52 per week. TGM separate, since its history is in the past.
parse Departures
End of explanation
"""


SD={}
SC=json.loads(file('../json/SC2.json','r').read())

#pop out last - if applicable
try: SD.pop(c)
except: pass
for h in range(len(AP.keys())):
    c=AP.keys()[h]
    #country not parsed yet
    if c in SC:
        if c not in SD:
            SD[c]=[]
            print h,c
            airportialinks=AP[c]
            sch={}
            #all airports of country, where there is traffic
            for i in airportialinks:
                if i in SC[c]:
                    print i,
                    if i not in sch:sch[i]={}
                    url=baseurl+airportialinks[i]
                    m=urlgetter(url)
                    for d in range (3,17):
                        #date not parsed yet
                        if d not in sch[i]:
                            url=baseurl+airportialinks[i]+'arrivals/201704'+str(d)
                            m=urlgetter(url)
                            soup = BeautifulSoup(m, "lxml")
                            #if there are flights at all
                            if len(soup.findAll('table'))>0:
                                sch[i][d]=pd.read_html(m)[0] 
                            else: print '--W-',d,
            SD[c]=sch
            print 

"""
Explanation: good dates
End of explanation
"""


dbpath='E:/Dropbox/Public/datarepo/aviation/' #large file db path
file(dbpath+"json/SD_arrv.json",'w').write(repr(SD))

I3=json.loads(file('../json/I3.json','r').read())

MDF=pd.DataFrame()

for c in SD:
    sch=SD[c]
    mdf=pd.DataFrame()
    for i in sch:
        for d in sch[i]:
            df=sch[i][d].drop(sch[i][d].columns[3:],axis=1).drop(sch[i][d].columns[0],axis=1)
            df['To']=i
            df['Date']=d
            mdf=pd.concat([mdf,df])
    if len(sch)>0:
        mdf['City']=[i[:i.rfind(' ')] for i in mdf['From']]
        mdf['Airport']=[i[i.rfind(' ')+1:] for i in mdf['From']]
        cpath=I3[c].lower()
        file('../countries/'+cpath+"/json/mdf_arrv.json",'w').write(json.dumps(mdf.reset_index().to_json()))
        MDF=pd.concat([MDF,mdf])
        print c,

dbpath='E:/Dropbox/Public/datarepo/aviation/' #large file db path
MDF.reset_index().to_json(dbpath+'json/MDF_arrv.json')

"""
Explanation: Save
End of explanation
"""


