import os
import urllib
import webbrowser
import pandas as pd
from bs4 import BeautifulSoup

"""
Explanation: Python for Webscraping

SOC 590: Big Data and Population Processes
17th October 2016

Tutorial 2: Webscraping with a function
Outline

Import modules
Examine html structure of a webpage
Use Beautiful Soup within a function
Analyze .csv of webpages as a Pandas DataFrame
Process data with regular expressions

Visualize data with Altair


import relevant modules

standard library modules:
os
urllib
webbrowser


open source modules:
pandas
Beautiful Soup
End of explanation
"""


url = 'http://grad-schools.usnews.rankingsandreviews.com/best-graduate-schools/top-humanities-schools/sociology-rankings/page+1'
webbrowser.open_new_tab(url)

"""
Explanation: open US News Rankings for Sociology webpage
view page source to see html
End of explanation
"""


def extract_page_data(table_rows):
    """
    Extract and return the the desired information from the td elements within
    the table rows.
    """
    # create the empty list to store the player data
    # <tr> tag: defines a row in a table 
    # <td> tag: defines a cell in a table
    rows = []
    for row in soup.findAll('tr'):
        rows.append([val.text for val in row.find_all('td')])
        
    return rows[1:]


"""
Explanation: create a function to extract page data from US News
End of explanation
"""


us_news_rankings = []
errors_list = []

"""
Explanation: make empty lists for US News Rankings and errors
End of explanation
"""


url_template = 'http://grad-schools.usnews.rankingsandreviews.com/best-graduate-schools/top-humanities-schools/sociology-rankings/page+{page_number}'

# for each page from 1 to (and including) 4
for page_number in range(1, 5): 
    
    # Use try/except block to catch and inspect any urls that cause an error
    try:
        # get the webpage url
        url = url_template.format(page_number=page_number)

        # get the html
        html = urllib.request.urlopen(url)

        # create the BeautifulSoup object
        soup = BeautifulSoup(html, "lxml") 

        # get the column headers
        headers = [header.text for header in soup.findAll('th')]
        
        # start etracting rows
        table_rows = soup.select('td')[1:]
        
        school_data = extract_page_data(table_rows)
        
        # create the dataframe for the current page
        school_df = pd.DataFrame(school_data, columns=headers)
        school_df = pd.DataFrame(school_data)

        # append the current dataframe to the list of dataframes
        us_news_rankings.append(school_df)
    
    except Exception as e:
        # Store the url and the error it causes in a list
        error =[url, e] 
        # then append it to the list of errors
        errors_list.append(error)
    

"""
Explanation: go through multiple web pages of US News Rankings
End of explanation
"""


print(len(errors_list))
errors_list

"""
Explanation: view possible urls that returned errors
End of explanation
"""


us_news_rankings

us_news_df_raw = pd.concat(us_news_rankings, axis=0)
column_headers = ["rank", "school", "score"]
us_news_df_raw.columns = column_headers

us_news_df_raw.head(10)

"""
Explanation: view list of appended dataframes
concatenate dataframes into one .csv
End of explanation
"""


if not os.path.exists('../data/raw_data'):
    os.makedirs('../data/raw_data')

if not os.path.exists('../data/clean_data'):
    os.makedirs('../data/clean_data')
    

# Write out the raw rankings data to the raw_data folder in the data folder
us_news_df_raw.to_csv("../data/raw_data/us_news_rankings_RAW.csv", index=False)

"""
Explanation: make folders for raw_data and clean_data
End of explanation
"""


us_news_df_raw = pd.read_csv("../data/raw_data/us_news_rankings_RAW.csv")
us_news_df_raw = us_news_df_raw[0:len(us_news_df_raw)]
us_news_df_raw.head()

"""
Explanation: read in raw .csv of US News Rankings with Pandas
End of explanation
"""


us_news_df_raw["school_location"] = "NaN"
us_news_df_raw["school_name"] = "NaN"
us_news_df_raw["rank"] = us_news_df_raw.loc[:,('rank')].replace(r"\D",  "", regex = True)
us_news_df_raw["score"] = us_news_df_raw.loc[:,('score')].str.extract("(\d.\d)", expand=False)

us_news_df_raw.head(10)

"""
Explanation: make new columns for school location and school name
clean rank and score columns with regular expressions
End of explanation
"""


for i in range(0,len(us_news_df_raw)+1):
    try: 
        us_news_df_raw["school_name"][i] = us_news_df_raw["school"].str.split("\n\n")[i][0]
        us_news_df_raw["school_location"][i] = us_news_df_raw["school"].str.split("\n\n")[i][1]
    except:
        us_news_df_raw["school_name"][i] = "NaN"
        us_news_df_raw["school_location"][i] = "NaN"
us_news_df_raw["school_name"] = us_news_df_raw.loc[:,('school_name')].replace(r"\n",  "", regex = True)
us_news_df_raw["school_location"] = us_news_df_raw.loc[:,('school_location')].replace("\n",  "", regex = True)

cols = ["rank", "school_name", "school_location", "score"]
us_news_df_raw = us_news_df_raw[cols]
us_news_df_raw.head()

"""
Explanation: fill in the school name and school location columns with regular expressions
split the school column to create two new columns
End of explanation
"""


us_news_df_clean = us_news_df_raw[us_news_df_raw['school_name']!="NaN"]
us_news_df_clean.head()

"""
Explanation: drop observations that are empty rows (NaN observations on school name column)
End of explanation
"""


us_news_df_clean.to_csv("../data/clean_data/us_news_rankings_clean.csv")

from geopy.geocoders import Nominatim

geolocator = Nominatim()
locations = us_news_df_clean['school_location'].apply(lambda x: geolocator.geocode(x)) # equiv to df.sum(0)
address,coordinates  = zip(*locations)
latitude,longitude = zip(*coordinates)

us_news_df_clean.loc[:,'latitude'] = latitude
us_news_df_clean.loc[:,('longitude')] = longitude
us_news_df_clean = us_news_df_clean.apply(pd.to_numeric, errors="ignore")
us_news_df_clean.head()

us_news_df_clean['quintile'] = pd.qcut(us_news_df_clean['score'], 5, labels=False)

from altair import Chart, X, Y, Axis, Scale

Chart(us_news_df_clean).mark_circle(
    size=100,
    opacity=0.6
).encode(
    x=X('longitude:Q', axis=Axis(title=' ')),
    y=Y('latitude:Q', axis=Axis(title=' ')),
        #scale=Scale(domain=(-60, 80))),
    color='quintile:N',
).configure_cell(
    width=800,
    height=350
).configure_axis(
    grid=False,
    axisWidth=0,
    tickWidth=0,
    labels=False,

)

"""
Explanation: save clean .csv to file
End of explanation
"""


import sys
import altair
import bs4
print("System and module version information: \n")
print('Python version:', sys.version_info)
print('urllib.request version:', urllib.request.__version__)
print('pandas version:', pd.__version__)
print('altair version:',altair.__version__)
print('Beautiful Soup version:', bs4.__version__)

"""
Explanation: Review
In this tutorial, we learned how to examine the html structure of webpage and use a function based on the Beautiful Soup module to parse tables on multiple webpage into .csv. After creating a .csv of all webpage tables, we analyzed the .csv using the Pandas module. Lastly, we created vizualizations with the Altair library
End of explanation
"""


