reportModel = 133685247
full_form_url = 'http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.filter?ww_i_reportModel=133685247'
r = requests.get(full_form_url)
soup = BeautifulSoup(r.text, 'html.parser')


# get the name of the dropdown menus
select = soup.find_all('select')
select_name = [s.attrs['name'] for s in select]
select_name

select_fields = [soup.find('select',{'name': name}) for name in select_name]
# the html for each <select> field

# find the value for the informatique section
unite_acad_options = select_fields[0].find_all('option')
#unite_acad_options

unite_acad_informatique ={opt['value']: opt.text for opt in unite_acad_options if opt.text == 'Informatique'}
unite_acad_informatique

# periode academic
#select_fields[1].find_all('option')
period_acad = {opt['value']: opt.text for opt in select_fields[1].find_all('option') if opt['value'] != 'null' and int(opt.text.split('-')[0]) >= 2007}
period_acad

# get all the pedagogic periods
option = select_fields[2].find_all('option')
period_pedago = {opt['value']: opt.text for opt in option if opt.text != '' }
period_pedago

option = select_fields[3].find_all('option')
hiverEte = {opt['value']: opt.text for opt in option if opt['value'] != 'null'}
hiverEte

"""
Explanation: Collect the data
Get the select options
End of explanation
"""


# arguments are tuples (key, 'description') eg: ('2936286': "Semestre d'automne")
def collect_dataframe(t_unite_acad, t_periode_acad, t_periode_pedago, t_hiver_ete, final_headers):
    print("collect_dataframe: input: "+str(t_unite_acad)+" & "+str(t_periode_acad)+" &"+str(t_periode_pedago)+" & "+str(t_hiver_ete))

    #Send request
    params = {
                'ww_x_GPS': -1, 
                'ww_i_reportModel': reportModel, 
                'ww_i_reportModelXsl': 133685270, 
                'ww_x_UNITE_ACAD': t_unite_acad[0], 
                'ww_x_PERIODE_ACAD': t_periode_acad[0], 
                'ww_x_PERIODE_PEDAGO': t_periode_pedago[0], 
                'ww_x_HIVERETE': t_hiver_ete[0]
             }
    
    url = 'http://isa.epfl.ch/imoniteur_ISAP/!GEDPUBLICREPORTS.html'
    
    r = requests.get(url, params=params)
    soupe = BeautifulSoup(r.text, 'html.parser')
    
    # get all the tr tags
    tr_tags = soupe.find_all('tr')
    
    #Temporary dictionary that will collect all the entry of the dataframe
    data = {}
    
    # there may be several tables.
    current_table = 't1'
    
    # for each tr tag, determine if it is a table title, a header (ignore those) or a student row
    for tr in tr_tags:
        th = tr.find_all('th')
        if(len(th) == 1): #this is a table title
            current_table = th[0].text.split('\n')[0]
            data[current_table] = []
            #print('current table: '+str(current_table))
        
        elif(len(th) > 1): #this is the header row (ignore because is always the same)
            #print('headers: '+str([t.text for t in th]))
            pass
            
        else:
            # this is a student
            td_tags = tr.find_all('td')
            student = [td.text.replace('\xa0', ' ') for td in td_tags[:-1]] #drop last td because it is always empty
            # add the desired columns
            student.append(current_table)
            student.append(t_periode_acad[1])
            student.append(t_periode_pedago[1])
            #print('student: '+str(student))
            data[current_table].append(student)
    
    # return all different dataframes
    dframes = [pd.DataFrame(data[k], columns=final_headers) for k in list(data.keys())]

    return dframes
    

"""
Explanation: Collect
End of explanation
"""


perm_list = list(itertools.product(list(unite_acad_informatique.items()), list(period_acad.items()), list(period_pedago.items())))


# get the data
header = ['Civilité', 'Nom_Prénom', 'Orientation_Bachelor', 'Orientation_Master', 'Spécialisation', 'Filière_opt.', 'Mineur', 'Statut', 'Type_Echange', 'Ecole_Echange', 'No_Sciper', 'title', 'periode_acad', 'periode_pedago']
dframes = []
for (ua, pa, pp) in perm_list:
    res = collect_dataframe(ua, pa, pp,('null', 'null'), header)
    [dframes.append(df) for df in res]


#concatenate
all_data = pd.concat([df for df in dframes])

# write to file
#all_data.to_csv('all_data.csv')

len(all_data)

"""
Explanation: make one request for all permutations of (unite_acad, periode_acad, periode_pedago). We ignore hiverete because it is redundant.
End of explanation
"""


all_data['periode_pedago'].unique()

"""
Explanation: Note that the Mineur semestre X are always empty. That is why they dont appear here.
End of explanation
"""


