"""
# Web Scraping using Python
"""
"""
## Import Libraries
"""
import requests
from bs4 import BeautifulSoup
import pandas as pd
import numpy as np
# Request the webpage and get the text
urlreq = requests.get('https://en.wikipedia.org/wiki/Comparison_of_North_American_ski_resorts')

# Extract the text from the requested webpage
url = urlreq.text

# Create a parse tree with BS using the html parser
soup = BeautifulSoup(url, 'html.parser')
# Transforming html table into a dataframe
dfs = pd.read_html(url)
df = dfs[4] # 5th line
df
df.head()
"""
## Data Cleaning and Manipulation
"""
"""
Web scraping was used to collect information from Wikipedia which was then transformed into a dataframe. However, this dataset is not available in the most useful format as is and must be cleaned before any analysis can be performed.  

"""
# Rename columns
df.rename(columns = {'Resort name and website':'Resort', 'Nearest city':'City', 'State/province':'State', 'Adult weekend lift ticket window price (USD)':'Lift ticket (USD)'}, inplace=True)
# Drop columns that are not necessary 
df = df.drop(['Date statistics updated'], axis=1)
# Check for duplicates
df.duplicated().sum()
# Check for missing values
# df.isna().sum()
df.isna().sum().sort_values(ascending=False)
# Check datatypes of columns
df.dtypes
"""
### Missing Values
"""
# Exclude rows with missing values from 'City' column 
df = df[df['City'].notna()] # only take values that are not missing
# Replace missing values in base elevation, total trails, total lifts, and vertical drop columns with mean of their respective column
df['Base elevation (ft)'].fillna(value=df['Base elevation (ft)'].mean(), inplace=True)
df['Vertical drop (ft)'].fillna(value=df['Vertical drop (ft)'].mean(), inplace=True)
df['Total trails'].fillna(value=df['Total trails'].mean(), inplace=True)
df['Total lifts'].fillna(value=df['Total lifts'].mean(), inplace=True)
# Change datatypes to 'int' to remove decimals
convert_dict = {'Base elevation (ft)': int, 'Total trails': int, 'Total lifts': int, 'Vertical drop (ft)': int}
# Instead of individual coloumns we use a dictionary to convert and reduce work
df = df.astype(convert_dict)
df
"""
### Special Characters
"""
# Removing special characters from 'Lift ticket' column and replacing missing values with mean
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].replace('$', '', regex=True)
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].fillna(-1)
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].astype(str)
replacers = {'-1':float('nan'), 'Private Club':float('nan'), 'Free':float('nan'), 'Temporarily Closed':float('nan'), '25/season':float('nan'), '84.95[70]':float('84.95'), 'Closed Temporarily':float('nan')}
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].replace(replacers)
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].astype(float)
df['Lift ticket (USD)'].fillna(value=df['Lift ticket (USD)'].mean(), inplace=True)

# Change datatype to 'int' to remove decimals
df['Lift ticket (USD)'] = df['Lift ticket (USD)'].astype(int)
df
# Removing special characters from 'Avg annual snowfall' column and replacing missing values with mean
replacers1 = {'220[21]':float('220'), '80[58]':float('80'), '132[63]':float('132'), '180[69]':float('180'), '100[87]':float('100'), '120[89]':float('120'), '168[100]':float('168'), '8,464[102]':float('8464'), '311`':float('311'), '9,422[102]':float('9422')}
df['Avg annual snowfall (in)'] = df['Avg annual snowfall (in)'].replace(replacers1)
df['Avg annual snowfall (in)'] = df['Avg annual snowfall (in)'].astype(float)
df['Avg annual snowfall (in)'].fillna(value=df['Avg annual snowfall (in)'].mean(), inplace=True)

# Change datatype to 'int' to remove decimals
df['Avg annual snowfall (in)'] = df['Avg annual snowfall (in)'].astype(int)
# Removing special characters from 'Skiable acreage' column and replacing missing values with mean
df['Skiable acreage'] = df['Skiable acreage'].replace(replacers1)
df['Skiable acreage'] = df['Skiable acreage'].astype(float)
df['Skiable acreage'].fillna(value=df['Skiable acreage'].mean(), inplace=True)

df
# Change datatype to 'int' to remove decimals
df['Skiable acreage'] = df['Skiable acreage'].astype(int)
# Removing special characters from 'Peak elevation' column and replacing missing values with mean
df['Peak elevation (ft)'] = df['Peak elevation (ft)'].replace(replacers1)
df['Peak elevation (ft)'] = df['Peak elevation (ft)'].astype(float)
df['Peak elevation (ft)'].fillna(value=df['Peak elevation (ft)'].mean(), inplace=True)
# Change datatype to 'int' to remove decimals 
df['Peak elevation (ft)'] = df['Peak elevation (ft)'].astype(int)                                         
# Confirming that all missing values are replaced
df.isna().sum().sort_values(ascending=False)
"""
# Creating the csv file
"""
# Exporting data to a CSV 
df.to_csv('ski_resort_analysis.csv', index=False)