import os
import urllib
import webbrowser
import pandas as pd
from datetime import datetime
from bs4 import BeautifulSoup

url_2018_mvp_finalist = "https://www.basketball-reference.com/awards/awards_2018.html"

"""
Explanation: NBA MVP Comparisons
Part 3

6 <sup>th</sup> March 2019

In the last two blog posts we:
1. Gathered the relevant data for all NBA MVPs
2. Evaluated several machine learning models to select the MVP and chose the Latent Discriminant Analysis model as the best predictor.
The goal of this blog post is to gather the data necessary to predict the 2018-2019 MVP
We will look select from the MVP finalists in 2017-2018
* MVP will likely be a finalist from the previous year
    * Is this a valid assumption?
Update
On May 17<sup>th</sup>, the NBA announced the 2018-2019 MVP finalists:
* Giannis Antetokounmpo, Milwaukee Bucks
* Paul George, Oklahoma City Thunder
* James Harden, Houston Rockets
* source:  https://www.nba.com/article/2019/05/17/finalists-2019-nba-awards
Outline

Import modules
Examine html structure of webpage
Use a function with Beautiful Soup to parse webpages into .csv
Analyze .csv of webpage as a Pandas DataFrame

Process the data


import relevant modules

standard library modules:
os
urllib
webbrowser
datetime


open source modules:
pandas
Beautiful Soup
End of explanation
"""


webbrowser.open_new_tab(url_2018_mvp_finalist)

"""
Explanation: examine webpage
End of explanation
"""


html_finalist = urllib.request.urlopen(url_2018_mvp_finalist)
soup_finalist = BeautifulSoup(html_finalist, "lxml") 

"""
Explanation: get the html
create the BeautifulSoup object
End of explanation
"""


column_headers_finalist = [th.get_text() for th in soup_finalist.find_all('th', limit=30)]
column_headers_finalist = [header for header in column_headers_finalist if len(header) != 0]
column_headers_finalist = column_headers_finalist[1:]

print(f"raw column names in finalist table: {column_headers_finalist}")
column_headers_finalist = [header for header in column_headers_finalist if header not in ('Shooting', 'Advanced', 'Per Game')][:-4]
print(f"formatted column names in finalist table: {column_headers_finalist}")
print(f"{len(column_headers_finalist)} columns in finalist table")

"""
Explanation: Extract the necessary values for the column headers from the table and store them as a list
End of explanation
"""


table_rows_finalist = soup_finalist.find("tbody").find_all("tr")
print(f"the subset soup object is of type: {type(table_rows_finalist)}")
table_rows_finalist[-1]

"""
Explanation: The data is found within the tr elements of the first tbody element
We want the elements from the 3rd row and on
take a look at the last row to examine data quality
End of explanation
"""


player_href = [href for href in table_rows_finalist[-1].find_all("a")][0]["href"]
player_href

"""
Explanation: to get a player's 2018-2019 statistical information, we can extract the player's web page from the BeautifulSoup object
End of explanation
"""


base_basketball_ref_url = "https://www.basketball-reference.com"

player_link = base_basketball_ref_url + player_href
player_link

webbrowser.open_new_tab(player_link)

"""
Explanation: however, the player link is just a stub url and needs a base url for basketball-reference.com appended to it to access the player's web page
End of explanation
"""


def extract_finalist_data(table_rows):
    """
    Extract and return the the desired information from the td elements within the table rows.
    :param: table_rows:  list of soup `tr` elements
    :return: list of player-year MVP finalist observations
    """
    
    base_basketball_ref_url = "https://www.basketball-reference.com"
    
    # create the empty list to store the player data
    player_data = []
    
    for row in table_rows:  # for each row do the following

        # Get the text for each table data (td) element in the row
        player_rank = [td.get_text() for td in row.find_all("th")]
        player_list = [td.get_text() for td in row.find_all("td")]
        player_href = [href for href in row.find_all("a")][0]["href"]
        player_link = [base_basketball_ref_url + player_href]        
        
        # there are some empty table rows, which are the repeated column headers in the table
        # we skip over those rows and and continue the for loop
        if not player_list:
            continue
            
        # Now append the individual player data to list of all player data
        
        player_info = player_rank+player_list+player_link
        player_data.append(player_info)
        
    return player_data

"""
Explanation: create a function to extract MVP finalist data
End of explanation
"""


extracted_finalist_2018_data = extract_finalist_data(table_rows_finalist)

mvp_finalist_2018_data  = pd.DataFrame(extracted_finalist_2018_data)
mvp_finalist_2018_data.columns = column_headers_finalist+["player_link"]
print(f"the MVP finalist dataframe has {mvp_finalist_2018_data.shape[0]} rows (player-year observations) and {mvp_finalist_2018_data.shape[1]} columns")
mvp_finalist_2018_data.tail(6)

"""
Explanation: extract the data we want
and then store it in a DataFrame
End of explanation
"""


player_test_table = "https://www.basketball-reference.com/players/h/hardeja01.html#per_game::none"
player_profile_request = urllib.request.urlopen(player_test_table)
player_profile_soup = BeautifulSoup(player_profile_request, "lxml")

column_headers_player = [th.get_text() for th in player_profile_soup.find_all('th', limit=30)]
column_headers_player = [header for header in column_headers_player if len(header) != 0]
column_headers_player = column_headers_player[1:]
print(f"the columns in the career data tables are: \n {column_headers_player} \n")
print(f"there are {len(column_headers_player)} total columns in the career table")

player_name = player_profile_soup.find_all("h1")[0].text
player_years_active = [th.get_text() for th in player_profile_soup.find_all("tbody")[0].find_all("th")]
player_career_data = [td.get_text() for td in player_profile_soup.find_all("td")]

"""
Explanation: Extract the necessary values for the column headers from the table and store them as a list
create the BeautifulSoup object
End of explanation
"""


print(f"player name: {player_name}")
print(f"player years active: {player_years_active}")
print(f"player career data: {player_career_data}")

"""
Explanation: let's examine the data to make sure it conforms with our expectations
End of explanation
"""


def slice_per(source, step):
    return [source[i::step] for i in range(step)]

"""
Explanation: the player career data is a list that includes every year the player was active
this list is not broken up by year


to create yearly player data from the player career data list, we need a function to seperate the list
we can use the following function is from StackOverflow
End of explanation
"""


sliced_player_data = slice_per(player_career_data, 29)
print(type(sliced_player_data))
player_career_df = pd.DataFrame(sliced_player_data).transpose()
player_career_df.columns = column_headers_player
player_career_df

"""
Explanation: every 29 items in the list represents a year of player data
we will split the player_career_data object every 29 steps (using the language of the slice_per function) to create a year of data


then we add the column headers to the dataframe as a sanity check that the information was extracted correctly
End of explanation
"""


def extract_career_data(player_link):
    """
    Extract and return the the desired information from the td elements within the table rows.
    :param: table_rows:  list of soup `tr` elements
    :return: list of player-year MVP finalist observations
    """
        
    player_profile_request = urllib.request.urlopen(player_link)

    # create the BeautifulSoup object
    player_profile_soup = BeautifulSoup(player_profile_request, "lxml") 
    extracted_player_data = [td for td in player_profile_soup.find_all("tbody")]

    player_name = player_profile_soup.find_all("h1")[0].text
    player_years_active = [th.get_text() for th in player_profile_soup.find_all("tbody")[0].find_all("th")]
    player_career_data = [td.get_text() for td in extracted_player_data[0].find_all("td")]
    
    # slice list of list (player_data into yearly lists)
    sliced_player_data = slice_per(player_career_data, 29)
    
    player_career_df = pd.DataFrame(sliced_player_data).transpose()
    player_career_df.insert(0, "Player", player_name)
    player_career_df.insert(1, "Year", player_years_active)
        
    return player_career_df


"""
Explanation: now we need a function to extract the player career data using the link to the player's profile as the input
End of explanation
"""


all_player_career_data = [extract_career_data(player_link) for player_link in mvp_finalist_2018_data["player_link"]]

"""
Explanation: use list comprehension to extract career data for each player link in mvp_finalist_2018_data
End of explanation
"""


mvp_finalist_2019_career_data = pd.concat(all_player_career_data, axis=0)
print(f"the MVP finalist dataframe has {mvp_finalist_2018_data.shape[0]} rows (player-year observations) and {mvp_finalist_2018_data.shape[1]} columns")

mvp_finalist_2019_career_data.head()

"""
Explanation: use Pandas concat function to store all finalist data in one DataFrame
store all finalist data in one DataFrame
End of explanation
"""


mvp_finalist_2019_career_data.columns = ["Player"] + ["Year"] + column_headers_player
mvp_finalist_2019_career_data.columns

"""
Explanation: rename columns by concatentating column_headers_player with the two new columns we added in our extract_career_data function
End of explanation
"""


mvp_finalist_2019_career_data.tail()

"""
Explanation: examine the mvp_finalist_2019_career_data data
End of explanation
"""


os.makedirs('../data/raw_data', exist_ok=True)
os.makedirs('../data/clean_data', exist_ok=True)

"""
Explanation: Now that we fixed up the necessary columns, let's write out the raw data to a CSV file.
End of explanation
"""


mvp_finalist_2019_career_data.to_csv("../data/raw_data/mvp_finalist_2019_career_data.csv", index=False)

"""
Explanation: Write out the career data for 2018 MVP finalists to the raw_data folder in the data folder
End of explanation
"""


mvp_finalist_2019_df_clean = pd.read_csv("../data/raw_data/mvp_finalist_2019_career_data.csv", encoding = "Latin-1")
mvp_finalist_2019_df_clean.head()

mvp_finalist_2019_df_clean.columns

"""
Explanation: Cleaning the Data

Now that we have the raw MVP data, we need to clean it up a bit for data exploration
End of explanation
"""


mvp_finalist_columns_dict = {'Player':'player', 'Year':'year',  'Age': 'age', 'Tm': 'team', 'Lg': 'league',
                             'POS': 'position', 'G': 'games_played', 'GS': 'games_started', 'MP': 'avg_minutes',
                             'FG': 'field_goals_made_per_game', 'FGA': 'field_goals_attempted_per_game',
                             'FG%': 'field_goal_pct', '3P': 'three_pt_fg_made_per_game', '3PA': 'three_pt_fg_attempted_per_game',
                             '3P%': 'three_pt_pct', '2P': 'two_pt_fg_made_per_game', '2PA': 'two_pt_fg_attempted_per_game',
                             '2P%': 'two_pt_fg_pct', 'eFG%': 'effective_fg_pct',  
                             'FT': 'free_throws_made_per_game', 'FTA': 'free_throws_attempted_per_game',
                             'FT%': 'free_throw_pct', 'ORB': 'offensive_rebounds_per_game', 'DRB': 'defensive_rebounds_per_game',
                             'TRB': 'total_rebounds_per_game', 'AST': 'assists_per_game', 'STL': 'steals_per_game', 
                             'BLK': 'blocks_per_game', 'TOV': 'turnovers_per_game',  'PF': 'fouls_committed_per_game', 'PTS': 'points_per_game'
                              }

mvp_finalist_2019_df_clean.rename(index=str,columns=mvp_finalist_columns_dict, inplace=True)
mvp_finalist_2019_df_clean.head()

mvp_finalist_2019_df_clean.columns

"""
Explanation: create dictionaries for renaming columns
rename all columns with dictionaries
End of explanation
"""


mvp_finalist_2019_df_clean = mvp_finalist_2019_df_clean.apply(pd.to_numeric, errors="ignore")
mvp_finalist_2019_df_clean.info()

"""
Explanation: Cleaning Up the Rest of the Data

convert the data to proper numeric types
End of explanation
"""


num_cols_finalist = mvp_finalist_2019_df_clean.columns[mvp_finalist_2019_df_clean.dtypes != object]
mvp_finalist_2019_df_clean.loc[:, num_cols_finalist] = mvp_finalist_2019_df_clean.loc[:, num_cols_finalist].fillna(0)
mvp_finalist_2019_df_clean.info()

"""
Explanation: Get the column names for the numeric columns
Replace all NaNs with 0
End of explanation
"""


mvp_finalist_2019_df_clean = mvp_finalist_2019_df_clean[pd.notnull(mvp_finalist_2019_df_clean['player'])]
mvp_finalist_2019_df_clean.sort_values(['year'], ascending=False, axis=0, inplace=True)

mvp_finalist_2019_df_clean.to_csv("../data/clean_data/mvp_finalist_2019_df_clean.csv", index=False)
print(f" the dimensions for the final data are: {mvp_finalist_2019_df_clean.shape} (rows, columns)")
mvp_finalist_2019_df_clean.head()

"""
Explanation: We are finally done cleaning the data and now we can save it to a CSV file.
End of explanation
"""


import sys
import bs4
print(f'last updated: {datetime.now().strftime("%Y-%m-%d %H:%M")} \n')
print(f"System and module version information: \n")
print(f"Python version: {sys.version_info}")
print(f"urllib.request version: {urllib.request.__version__}")
print(f"pandas version: {pd.__version__}")
print(f"Beautiful Soup version: {bs4.__version__}")

"""
Explanation: Review

In this tutorial, we learned how to: 
examine the html structure of webpage 
use functions based on the Beautiful Soup module to parse tables on multiple webpage into .csv
analyzed a .csv file using the Pandas module



Download this notebook or see a static view here
End of explanation
"""


