# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: DataX Python3.6
#     language: python
#     name: python3
# ---

# +
import pandas as pd

# tables = pd.read_html("https://w5.ab.ust.hk/wcq/cgi-bin/1830/subject/COMP")

# print(tables[1])

# +
# tables[3]
# every odd number table in each major page
# -

majorList = [
'ACCT',
'AESF',
'BIBU',
'BIEN',
'BIPH',
'BTEC',
'CBME',
'CENG',
'CHEM',
'CHMS',
'CIEM',
'CIVL',
'COMP',
'CPEG',
'CSIC',
'CSIT',
'ECON',
'EEMT',
'EESM',
'ELEC',
'EMBA',
'ENEG',
'ENGG',
'ENTR',
'ENVR',
'ENVS',
'EVNG',
'EVSM',
'FINA',
'GBUS',
'GFIN',
'GNED',
'HART',
'HHMS',
'HLTH',
'HMMA',
'HUMA',
'IBTM',
'IDPO',
'IEDA',
'IIMP',
'IMBA',
'ISDN',
'ISOM',
'JEVE',
'LABU',
'LANG',
'LIFS',
'MAED',
'MAFS',
'MARK',
'MATH',
'MECH',
'MESF',
'MGCS',
'MGMT',
'MILE',
'MIMT',
'MSBD',
'NANO',
'OCES',
'PDEV',
'PHYS',
'PPOL',
'RMBI',
'SBMT',
'SCIE',
'SHSS',
'SOSC',
'SSMA',
'SUST',
'TEMG',
'UROP',
'WBBA']

tableMajorList = []
for course in majorList:
    url = "https://w5.ab.ust.hk/wcq/cgi-bin/1830/subject/"+course
    insertTables = pd.read_html(url)
    tableMajorList.append(insertTables)

# +
# # each item
# dftest = tableMajorList[0][1]
# dftest.columns = dftest.iloc[0]
# # dftest.reindex(dftest.index.drop(1))
# dftest = dftest[1:]
# dftest

# +
# tableMajorList[7][1].ix[1:,1]

# +
# combine all the tables together, given they have same column name

majortableDFlist = []
checkList = ['Section', 'Date & Time', 'Room', 'Instructor', 'Quota', 'Enrol', 'Avail', 'Wait', 'Remarks']
for majorindex in range(len(tableMajorList)):
    for tableindex in range(len(tableMajorList[majorindex])):
        dftest = tableMajorList[majorindex][tableindex]
        dftest.columns = dftest.iloc[0]
        dftest = dftest[1:]
#         print(dftest.columns)
#         if len(dftest.columns) == len(checkList):
        if list(dftest.columns) == checkList:
            majortableDFlist.append(dftest)
# -

# majortableDFlist[1]
concat_df = pd.concat(majortableDFlist)

concat_df_backup = concat_df

# +
weekdays = ['Mo','Tu','We','Th','Fr']
index=0

timelist = []
roomlist = []
# droplistIndex = []

for i in concat_df_backup['Section']:
#     boolcheck = False
    for j in weekdays:
        # in case that time is on Section column 
        if j in i:
#             print(concat_df_backup.iloc[index])#['Section'])
            timelist.append(concat_df_backup.iloc[index]['Section'])
            roomlist.append(concat_df_backup.iloc[index]['Date & Time'])
#             boolcheck = True
#             print(boolcheck)
#             droplistIndex.append(index)
#             print(index)
        # in case that time is on Time column
#         if j not in i:
#             if boolcheck == False:
#                 print(boolcheck)
#                 timelist.append(concat_df_backup.iloc[index]['Date & Time'])
#                 roomlist.append(concat_df_backup.iloc[index]['Room'])
    index+=1

print(len(timelist))    

index=0
for i in concat_df_backup['Date & Time']:
    for j in weekdays:
        if j in i:
            timelist.append(concat_df_backup.iloc[index]['Date & Time'])
            roomlist.append(concat_df_backup.iloc[index]['Room'])
    index+=1
    
print(len(timelist))
# -

completeDF = pd.DataFrame()
completeDF['Date & Time'] = timelist
completeDF['Room'] = roomlist
# completeDF = completeDF[~completeDF['Date & Time'].str.contains("TBA")]
# completeDF = completeDF[~completeDF['Room'].str.contains("TBA")]
# completeDF[completeDF['Date & Time']!="TBA"]
completeDF

# +
# line = completeDF.iloc[10][0]
# print(line)
# line.split(sep='Mo ')[-1]

# seperators = ['MoWeFr ', 'MoWe ', 'WeFr ','TuTh ', 'Tu ','Fr ','Mo ','Th ','We ']

# weekdayList = []
# actualTimeList = []
# for i in range(len(completeDF)):
#     doneBool = False
#     line = completeDF.iloc[i][0]
#     for sep in seperators:
# #         print(sep)
#         if doneBool == False:
#             if sep in line:
#                 time = line.split(sep='sep')[-1]
#                 actualTimeList.append(time)
#                 doneBool = True
#                 weekdayList.append(sep)
                
seperators = ['MoWeFr ', 'MoWe ', 'WeFr ','TuTh ', 'Tu ','Fr ','Mo ','Th ','We ']

weekdayList = []
actualTimeList = []
properroomList = []

for i in range(len(completeDF)):
    line = completeDF.iloc[i][0]
    
    if 'MoWeFr ' in line:
        time = line.split(sep='MoWeFr ')[-1]
        actualTimeList.append(time)
        weekdayList.append('Mo')
        properroomList.append(completeDF.iloc[i][1])
        actualTimeList.append(time)
        weekdayList.append('We')
        properroomList.append(completeDF.iloc[i][1])
        actualTimeList.append(time)
        weekdayList.append('Fr')
        properroomList.append(completeDF.iloc[i][1])

    if 'MoWe ' in line:
        time = line.split(sep='MoWe ')[-1]
        actualTimeList.append(time)
        weekdayList.append('Mo')
        properroomList.append(completeDF.iloc[i][1])
        actualTimeList.append(time)
        weekdayList.append('We')
        properroomList.append(completeDF.iloc[i][1])
        
    if 'WeFr ' in line:
            time = line.split(sep='WeFr ')[-1]
            actualTimeList.append(time)
            weekdayList.append('We')
            properroomList.append(completeDF.iloc[i][1])
            actualTimeList.append(time)
            weekdayList.append('Fr')
            properroomList.append(completeDF.iloc[i][1])
            
    if 'TuTh ' in line:
            time = line.split(sep='TuTh ')[-1]
            actualTimeList.append(time)
            weekdayList.append('Tu')
            properroomList.append(completeDF.iloc[i][1])
            actualTimeList.append(time)
            weekdayList.append('Th')
            properroomList.append(completeDF.iloc[i][1])

    if 'Mo ' in line:
        if 'MoWeFr ' not in line:
            if 'MoWe ' in line:
                time = line.split(sep='Mo ')[-1]
                actualTimeList.append(time)
                weekdayList.append('Mo')
                properroomList.append(completeDF.iloc[i][1])
                
    if 'We ' in line:
        if 'WeFr ' not in line:
            if 'MoWe ' in line:
                time = line.split(sep='We ')[-1]
                actualTimeList.append(time)
                weekdayList.append('We')
                properroomList.append(completeDF.iloc[i][1])
                
    if 'Fr ' in line:
        if 'MoWeFr ' not in line:
            if 'WeFr ' in line:
                time = line.split(sep='Fr ')[-1]
                actualTimeList.append(time)
                weekdayList.append('Fr')
                properroomList.append(completeDF.iloc[i][1])
                
    if 'Tu ' in line:
        if 'TuTh ' not in line:
            time = line.split(sep='Tu ')[-1]
            actualTimeList.append(time)
            weekdayList.append('Tu')
            properroomList.append(completeDF.iloc[i][1])
            
    if 'Th ' in line:
        if 'TuTh ' not in line:
            time = line.split(sep='Th ')[-1]
            actualTimeList.append(time)
            weekdayList.append('Th')
            properroomList.append(completeDF.iloc[i][1])
# -

# completeDF['sep'] = weekdayList
# completeDF['actual time'] = actualTimeList
finalResults = pd.DataFrame()
finalResults['day'] = weekdayList
finalResults['time'] = actualTimeList
finalResults['room'] = properroomList

finalResults[finalResults['room']!='TBA']

finalResults.to_csv("emptyClassRoom_timetable.csv")

# ### ignore below

totalClassroomList = finalResults['room'].unique()

alltimerangeList = finalResults['time'].unique()

# +
# # if for a certain classroom, there is no matching time range with a given weekday requirement, it is an empty classroom
# weekdays = ['Mo','Tu','We','Th','Fr']
# # emptyCounter = 0

# xdayList = []
# xtimeList = []
# xroomList = []

# for weekday in weekdays:
#     for classroom in totalClassroomList:
#         classdf = finalResults[finalResults['room']==classroom]
# #         print(len(classdf))
#         for time in alltimerangeList:
# #             print(time)
#             if time not in list(classdf['time']):
# #                 print(time)
# #         for time in classdf['Date & Time']:
# #             if time not in alltimerangeList:
# #                 emptyCounter+=1
# #                 print("LOL")
#                 if weekday in time:
#                     xdayList.append(weekday)
#                     xtimeList.append(time)
#                     xroomList.append(classroom)
#                     print("smght")
                
                
# #     if emptyCounter==0:
# #     if emptyCounter==0:
# #         print(weekday)
# #         print(classroom)
# #         print(time)

# +
# import itertools
# if for a certain classroom, there is no matching time range with a given weekday requirement, it is an empty classroom
weekdays = ['Mo','Tu','We','Th','Fr']
# emptyCounter = 0

xdayList = []
xtimeList = []
xroomList = []

for weekday in weekdays:
    for classroom in totalClassroomList:
        classdf = finalResults[finalResults['room']==classroom]
#         for index, row in classdf.iterrows():
#             if row['time'] not in alltimerangeList:
        for time in alltimerangeList:
            if time not in classdf['time']:
                xdayList.append(weekday)
                xtimeList.append(time)
                xroomList.append(classroom)
#                 print("smght")
                    
            
# -

totalClassroomList

alltimerangeList

finalResults2 = pd.DataFrame()
finalResults2['day'] = xdayList
finalResults2['time'] = xtimeList
finalResults2['room'] = xroomList

finalResults2

# +
# finalResults_filtered = finalResults[finalResults['room'] != 'TBA']
# -

finalResults_filtered[finalResults_filtered['day']=='Mo']

alltimerangeList

StarcoderdataJupyterScriptsDedupFiltered