"""
<a id = "1"></a><h1 id="Salary Prediction with Machine Learning"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Salary Prediction with Machine Learning</span></h1>

This dataset was originally taken from the StatLib library which is maintained at Carnegie Mellon University. This is part of the data that was used in the 1988 ASA Graphics Section Poster Session. The salary data were originally from Sports Illustrated, April 20, 1987. The 1986 and career statistics were obtained from The 1987 Baseball Encyclopedia Update published by Collier Books, Macmillan Publishing Company, New York.
"""
"""
![expected-salary-at-08-31-2016.jpg](attachment:8cb1b7af-96ac-4981-bd24-7586e05c2896.jpg)
"""
"""
- A data frame with 322 observations of major league players on the following 20 variables.
- AtBat: Number of times at bat in 1986
- Hits: Number of hits in 1986
- HmRun: Number of home runs in 1986
- Runs: Number of runs in 1986
- RBI: Number of runs batted in in 1986
- Walks: Number of walks in 1986
- Years: Number of years in the major leagues
- CAtBat: Number of times at bat during his career
- CHits: Number of hits during his career
- CHmRun: Number of home runs during his career
- CRuns: Number of runs during his career
- CRBI: Number of runs batted in during his career
- CWalks: Number of walks during his career
- League: A factor with levels A and N indicating player’s league at the end of 1986
- Division: A factor with levels E and W indicating player’s division at the end of 1986
- PutOuts: Number of put outs in 1986
- Assists: Number of assists in 1986
- Errors: Number of errors in 1986
- Salary: 1987 annual salary on opening day in thousands of dollars
- NewLeague: A factor with levels A and N indicating player’s league at the beginning of 1987

**Number of Observation Units: 322**

**Variable Number: 20**


"""
"""
<a id = "1"></a><h1 id="Data Preprocessing"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Data Preprocessing</span></h1>

"""
import numpy as np
import pandas as pd 
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.metrics import mean_squared_error, r2_score
import matplotlib.pyplot as plt
from sklearn.preprocessing import scale 
from sklearn.preprocessing import StandardScaler
from sklearn import model_selection
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn import neighbors
from sklearn.svm import SVR

from warnings import filterwarnings
filterwarnings('ignore')
df = pd.read_csv("../input/hitters/Hitters.csv")
df.head()
def check_df(dataframe, head=5):
    print("⛤ Shape ⛤")
    print(dataframe.shape)
    print("⛤ Types ⛤")
    print(dataframe.dtypes)
    print("⛤ Head ⛤")
    print(dataframe.head(head))
    print("⛤ Tail ⛤")
    print(dataframe.tail(head))
    print("⛤ NA ⛤")
    print(dataframe.isnull().sum())
    print("⛤ Quantiles ⛤")
    print(dataframe.quantile([0, 0.05, 0.50, 0.95, 0.99, 1]).T)
    
check_df(df)
df.nunique() # Number of unique observations in variables
df["League"].value_counts()  #categorical variable
df.League.value_counts().plot.barh();
df["NewLeague"].value_counts()
df.NewLeague.value_counts().plot.barh();
df["Division"].value_counts()
df.Division.value_counts().plot.barh();
df["Salary"].max()
df["Salary"].min()
import seaborn as sns
sns.distplot(df.Salary);
df.corr()
f, ax = plt.subplots(figsize= [15,15])
sns.heatmap(df.corr(), annot=True, fmt=".2f", ax=ax, cmap = "magma" )
ax.set_title("Correlation Matrix", fontsize=20)
plt.show()
df.sort_values("CHits",ascending = False)
"""
<a id = "1"></a><h1 id="Means"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Means</span></h1>
"""
df.groupby("League").agg({"Salary": "mean"})
df.groupby("NewLeague").agg({"Salary": "mean"})
df.groupby("Division").agg({"Salary": "mean"})
df.groupby("CRBI").agg({"Salary": "mean"})
df.groupby("CRuns").agg({"Salary": "mean"})
df.groupby(["League","Years"]).agg({"Salary": "mean"})
df.groupby(["NewLeague","Years"]).agg({"Salary": "mean"})
df.groupby(["Division","Years"]).agg({"Salary": "mean"})
df.groupby("League").agg({"Errors": "mean"})
df.groupby("League").agg({"CAtBat": "mean"})
"""
<a id = "1"></a><h1 id="Max"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Max</span></h1>
"""
df.groupby("League").agg({"CHits": "max"})
df.groupby("Division").agg({"CHits": "max"})
df.groupby("League").agg({"Hits": "max"})
df.groupby("League").agg({"AtBat": "max"})
df.groupby("League").agg({"Years": "max"})
df.groupby("League").agg({"Errors": "max"})
df.groupby("League").agg({"PutOuts": "max"})
df.groupby("League").agg({"Assists": "max"})
df.groupby("Years").agg({"CAtBat": "max"})
df.groupby(["League", "Years"]).agg({"CAtBat": "max"})
"""
<a id = "1"></a><h1 id="Feature Extraction"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Feature Extraction</span></h1>
"""
df["OrtCAtBat"] = df["CAtBat"] / df["Years"]
df["OrtCHits"] = df["CHits"] / df["Years"]
df["OrtCHmRun"] = df["CHmRun"] / df["Years"]
df["OrtCruns"] = df["CRuns"] / df["Years"]
df["OrtCRBI"] = df["CRBI"] / df["Years"]
df["OrtCWalks"] = cwalks = df["CWalks"] / df["Years"]
df.head()
df = df.drop(['AtBat','Hits','HmRun','Runs','RBI','Walks','Assists','Errors',"PutOuts",'League','NewLeague', 'Division'], axis=1)
df.head()
df.isnull().sum()
import missingno as msno
msno.bar(df);
df_missing = df[df["Salary"].isnull()].head()
df_missing
from sklearn.impute import KNNImputer
imputer = KNNImputer(n_neighbors = 4)
df_filled = imputer.fit_transform(df)
#We fill in the missing observations with the KNN method.
df = pd.DataFrame(df_filled,columns = df.columns)
df.isnull().sum()
import seaborn as sns
sns.boxplot(x = df["Salary"]);
df["Salary"].describe()
for feature in df:

    Q1 = df[feature].quantile(0.25)
    Q3 = df[feature].quantile(0.75)
    IQR = Q3-Q1
    upper = Q3 + 1.5*IQR
    lower = Q1 - 1.5*IQR

    if df[(df[feature] > upper) | (df[feature] < lower)].any(axis=None):
        print(feature,"yes")
        print(df[(df[feature] > upper) | (df[feature] < lower)].shape[0])
    else:
        print(feature, "no")
Q1 = df.Salary.quantile(0.25)
Q3 = df.Salary.quantile(0.75)
IQR = Q3-Q1
lower = Q1 - 1.5*IQR
upper = Q3 + 1.5*IQR
df.loc[df["Salary"] > upper,"Salary"] = upper
import seaborn as sns
sns.boxplot(x = df["Salary"]);
from sklearn.neighbors import LocalOutlierFactor
lof =LocalOutlierFactor(n_neighbors= 10)
lof.fit_predict(df)
df_scores = lof.negative_outlier_factor_
np.sort(df_scores)[0:30]

threshold = np.sort(df_scores)[7]
threshold
outlier = df_scores > threshold
df = df[outlier]
df.shape
"""
<a id = "1"></a><h1 id="Machine Learning Methods"><span class="label label-default" style="background-color:#f5c0c0; font-size:30px; 
color: Black; ">Machine Learning Methods</span></h1>
"""
"""
<a id = "1"></a><br>
## 1 . KNN
"""
y = df["Salary"]
X = df.drop("Salary",axis=1)
X.head()
import statsmodels.api as sm
from sklearn.feature_selection import RFE
#Backward Elimination
cols = list(X.columns)
pmax = 1
while (len(cols)>0):
    p= []
    X_1 = X[cols]
    X_1 = sm.add_constant(X_1)
    model = sm.OLS(y,X_1).fit()
    p = pd.Series(model.pvalues.values[1:],index = cols)      
    pmax = max(p)
    feature_with_p_max = p.idxmax()
    if(pmax>0.05):
        cols.remove(feature_with_p_max)
    else:
        break
selected_features_BE = cols
print(selected_features_BE)

#https://towardsdatascience.com/feature-selection-with-pandas-e3690ad8504b
X = df[selected_features_BE]
X.head()
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X = scaler.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X, y, 
                                                    test_size=0.20, 
                                                    random_state=46)
knn_model = KNeighborsRegressor().fit(X_train, y_train)
y_pred = knn_model.predict(X_test)
knn_base = np.sqrt(mean_squared_error(y_test, y_pred))
knn_base
"""
#  Model Tuning
"""
knn_params = {"n_neighbors": np.arange(2,30,1)}

knn_model = KNeighborsRegressor()

knn_cv_model = GridSearchCV(knn_model, knn_params, cv = 10).fit(X_train, y_train)
knn_cv_model.best_params_
"""
# Final Model Tuning
"""
knn_tuned = KNeighborsRegressor(**knn_cv_model.best_params_).fit(X_train, y_train)
y_pred = knn_tuned.predict(X_test)
knn_final = np.sqrt(mean_squared_error(y_test, y_pred))
knn_final
"""
<a id = "1"></a><br>
## 2 . SVR
"""
svr_model = SVR().fit(X_train, y_train)
y_pred = svr_model.predict(X_test)
svr_base = np.sqrt(mean_squared_error(y_test, y_pred))
svr_base
"""
# Model Tuning
"""
svr_model = SVR() 

svr_params = {"C": [0.01,0.001, 0.2, 0.1,0.5,0.8,0.9,1, 10, 100, 500,1000]}

svr_cv_model = GridSearchCV(svr_model, svr_params, cv = 10, n_jobs = -1, verbose =  2).fit(X_train, y_train)
svr_cv_model.best_params_
svr_tuned = SVR(**svr_cv_model.best_params_).fit(X_train, y_train)
"""
# Final Model Tuning
"""
y_pred = svr_tuned.predict(X_test)
svr_final = np.sqrt(mean_squared_error(y_test, y_pred))
svr_final
"""
<a id = "1"></a><br>
## 3 . CART
"""
cart_model = DecisionTreeRegressor()
cart_model.fit(X_train, y_train)
y_pred = cart_model.predict(X_test)
cart_base = np.sqrt(mean_squared_error(y_test, y_pred))
cart_base
"""
# Model Tuning
"""
cart_model = DecisionTreeRegressor()
cart_params = {"max_depth": [2,3,4,5,10,20,100, 1000],
              "min_samples_split": [2,10,5,30,50,10]}
cart_cv_model = GridSearchCV(cart_model, cart_params, cv = 10, n_jobs = -1, verbose =  2).fit(X_train, y_train)
cart_cv_model.best_params_
cart_tuned = DecisionTreeRegressor(**cart_cv_model.best_params_).fit(X_train, y_train)
y_pred = cart_tuned.predict(X_test)
cart_final = np.sqrt(mean_squared_error(y_test, y_pred))
cart_final
"""
<a id = "1"></a><br>
## 4 . Random Forests
"""
rf_model = RandomForestRegressor(random_state = 42).fit(X_train, y_train)
y_pred = rf_model.predict(X_test)
rf_base = np.sqrt(mean_squared_error(y_test, y_pred))
rf_base
"""
# Model Tuning
"""
rf_params = {"max_depth": [5,10,None],
            "max_features": [2,5,10],
            "n_estimators": [100, 500, 900],
            "min_samples_split": [2,10,30]}
rf_cv_model = GridSearchCV(rf_model, rf_params, cv = 10, n_jobs = -1, verbose = 2).fit(X_train , y_train)
rf_cv_model.best_params_
rf_tuned = RandomForestRegressor(**rf_cv_model.best_params_).fit(X_train, y_train)
y_pred = rf_tuned.predict(X_test)
rf_final = np.sqrt(mean_squared_error(y_test, y_pred))
rf_final
rf_tuned.feature_importances_
Importance = pd.DataFrame({'Importance':rf_tuned.feature_importances_*100}, 
                          index = ['CRuns', 'CWalks', 'OrtCAtBat', 'OrtCHits', 'OrtCRBI', 'OrtCWalks'])

Importance.sort_values(by = 'Importance', 
                       axis = 0, 
                       ascending = True).plot(kind = 'barh', 
                                              color = 'r', )

plt.xlabel('Variable Importance')
plt.gca().legend_ = None