import pandas as pd
import json
json_data = open('/home/yohna/Documents/quora_challenges/views/sample/input00.in') # Edit this to where you have put the input00.in file

data = []
for line in json_data:
    data.append(json.loads(line))

data.remove(9000)
data.remove(1000)

df = pd.DataFrame(data)

df['anonymous'] = df['anonymous'].map({False: 0, True:1}).astype(int)
cleaned_df=df[:9000]

# to make reading the question_text cells easier, remove the maximum column width 
pd.set_option('display.max_colwidth', -1) 

# Don't care about warnings in this notebook
import warnings
warnings.filterwarnings("ignore")

"""
Explanation: 1. Linear Regression
End of explanation
"""


data_df = cleaned_df[:9000]
#---------------------------------------------------------------
# The product of num_answers and context_present

# first define context_present
data_df['context_present'] = data_df['context_topic'].apply(lambda x: 0 if x==None else 1)
# then multiply and make it into a column
data_df['context_xnum_ans'] = data_df['context_present'] * data_df['num_answers']
#---------------------------------------------------------------

#---------------------------------------------------------------
# Questions with at least 29 Answers

data_df['num_ans>= 29'] = data_df['num_answers'].apply(lambda x: 1 if x>=29 else 0)
#---------------------------------------------------------------

#---------------------------------------------------------------
# The product of a boolean column of questions ...
# ... containing "top3" (see the preceeding notebook) words | corr_coef = 0.348

data_df['qcontains_comb_1'] = data_df['question_text'].apply(lambda x: 1 if any(pd.Series(x).str.contains('university|list|With|physics?|shown|have|single|finding|around|mind|Indian|come|interesting|most|inspired|guy|movies|value|instead|most?|movies?|indian|girlfriend?|advice|across?|physical|cross|not|hate|Apple|actually|found|modern|technology|there|biggest|India?|each|but|within|physics|tell|intelligence|girlfriend|changes|sound|Glass?|iOS|mind-blowing|universe?|India|cases|right|Why|boyfriend|true|efforts|facts|girl|some')) else 0)

data_df['qcontains_comb_1_xnum_ans'] = data_df['qcontains_comb_1'] * data_df['num_answers']
#---------------------------------------------------------------

#---------------------------------------------------------------
# The product of a boolean column of the number of characters of questions ...
# ... and the number of answers.  | corr_coef = 0.392

data_df['len_chars'] = data_df.question_text.apply(lambda x: len(x))

data_df['qnumchars_lessThan_182_xnum_ans'] = data_df['num_answers'] * data_df['len_chars'].apply(lambda x: 1 if x <= 182 else 0)
#---------------------------------------------------------------

#---------------------------------------------------------------
# The sum of number of followers of the topics the question is tagged for  | corr_coef = 0.121

def funn(x): #where x will be a row when running `apply`
    return sum(x[i]['followers'] for i in range(len(x)))
data_df['topics_followers'] = data_df['topics'].apply(funn)
data_df.drop(['topics'], axis =1, inplace=True)
#---------------------------------------------------------------

data_df.drop(['context_present', 'qcontains_comb_1'], axis=1, inplace=True) #To avoid multilinearity

"""
Explanation: First let's add the new features.
End of explanation
"""


data_df.head()

# numerical values only and 'question_key'
df_n = data_df[['__ans__', 'question_key', 'anonymous', 'num_answers', 'context_xnum_ans', 'num_ans>= 29', 'qcontains_comb_1_xnum_ans', 'len_chars', 'qnumchars_lessThan_182_xnum_ans', 'topics_followers']]

#set the index to question_key
df_n.set_index(df_n['question_key'], 
                  inplace=True
                 )
df_n.drop(['question_key'], axis=1, inplace=True)
# Since the new index has a name ("question_key"), pandas will add an extra row with
# the index name in the first entry with 0 in the rest. Fix this by setting the index
# name to None
df_n.index.name = None 

df_n.head()

"""
Explanation: 
End of explanation
"""


import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
sns.set(context="paper")
corrmat = df_n.corr()
f, ax = plt.subplots(figsize=(14, 10))
f.text(0.45, 0.93, "Correlation coefficients", ha='center')
sns.heatmap(corrmat, square=True, linewidths=0.01, cmap='coolwarm', annot=True)

"""
Explanation: 1.1 Training the Model
1.1.1 Correlation Matrix
End of explanation
"""


from sklearn.model_selection import train_test_split
y = df_n.__ans__ # the target column
x = df_n.drop('__ans__', axis=1)
x_train, x_test, y_train, y_test = train_test_split(x, y, 
                                                   test_size = 0.2,
                                                   random_state = 123)

x_train.head()

"""
Explanation: Should I have multiplied those features by num_answers?? I fear I might have induced multicollinearity.

1.1.2 Split to train and test sets
End of explanation
"""


# RobustScaler
from sklearn.preprocessing import RobustScaler
scaler = RobustScaler(quantile_range=(5,95)).fit(x_train)

X_train = scaler.transform(x_train)
X_test = scaler.transform(x_test) #it's okay not to do it now

"""
Explanation: 1.1.3 Pipeline
End of explanation
"""


xt_columns = list(df_n.columns)
xt_columns.remove('__ans__') # columns of X_train

xt_columns

X_train_df = pd.DataFrame(X_train, columns=xt_columns, index=x_train.index) # X_train and y_train are not dataframes initially
y_train_df = pd.DataFrame(y_train, columns=['__ans__'])
df_for_corr = pd.concat([y_train_df, X_train_df], axis=1)

df_for_corr.head()

sns.set(context="paper")
corrmat_1 = df_for_corr.corr()
f, ax = plt.subplots(figsize=(14, 10))
f.text(0.45, 0.93, "Correlation coefficients", ha='center')
sns.heatmap(corrmat_1, square=True, linewidths=0.01, cmap='coolwarm', annot=True)

"""
Explanation: 1.1.4 Correlation Heatmap on Scaled Data
End of explanation
"""


from sklearn.linear_model import LinearRegression
import numpy as np
from sklearn.metrics import mean_squared_error

lin_reg = LinearRegression()
lin_reg.fit(X_train, y_train)

y_pred_train_lin = lin_reg.predict(X_train)

lin_rmse_train = np.sqrt(mean_squared_error(y_pred_train_lin, y_train))

lin_rmse_train

"""
Explanation: 1.1.4 The Model: Linear Regressor
End of explanation
"""


y_pred_test_lin = lin_reg.predict(X_test)
lin_rmse_test = np.sqrt(mean_squared_error(y_pred_test_lin, y_test))
lin_rmse_test

df_n['__ans__'].describe()

y_train.values

from plotnine import *
(ggplot(x_test, aes(x=y_test.values, y=y_pred_test_lin)) + theme_bw() + geom_point() + geom_smooth())

min(abs((y_train.values - y_pred_train_lin)))

"""
Explanation: And on the test set:
End of explanation
"""


from sklearn.model_selection import cross_val_score
scores = cross_val_score(lin_reg, X_train, y_train,
                        scoring = 'neg_mean_squared_error', cv=10)
rmse_scores = np.sqrt(-scores)

rmse_scores.mean() #mean

rmse_scores.std()

df_n.__ans__.describe()

"""
Explanation: 1.2 Cross Validation
End of explanation
"""


from sklearn.tree import DecisionTreeRegressor

tree_reg = DecisionTreeRegressor()
tree_reg.fit(X_train, y_train)

y_train_tree_pred = tree_reg.predict(X_train)
tree_mse = mean_squared_error(y_train, y_train_tree_pred)
tree_rmse = np.sqrt(tree_mse)
tree_rmse

"""
Explanation: 1.3 Conclusion
Since, most of the target values lie between 0.458 and 4.02, our model, which has 35.42 rmse cv score, is still underfitting. So our next option is to either select a more powerful model, or to reduce the constraints (aka hyperparameters) on the model. But, this linear regression model is not regularized, so it must be that we haven't explored the features well, or have to go with a more powerful model.
On this challenge, I don't wish to spend more time on the feature engineering than I already have. So, I will resort to more powerful models. 

2. Decision Tree Regression
End of explanation
"""


scores_tree = cross_val_score(tree_reg, X_train, y_train,
                             scoring='neg_mean_squared_error', cv=10)
rmse_scores_tree = np.sqrt(-scores_tree)
print('mean:', rmse_scores_tree.mean(), 'and', 'std:', rmse_scores_tree.std())

rmse_scores_tree

"""
Explanation: This is quite impressive! But it could also be that the model is overfitting, so let's do cross validation.
End of explanation
"""


y_test_tree_pred = tree_reg.predict(X_test)
tree_mse_test = mean_squared_error(y_test, y_test_tree_pred)
tree_rmse_test = np.sqrt(tree_mse_test)
tree_rmse_test

"""
Explanation: Yes, our fears are real. It overfit.
End of explanation
"""


from sklearn.ensemble import RandomForestRegressor

forest_reg = RandomForestRegressor()
forest_reg.fit(X_train, y_train)

y_train_forest_pred = forest_reg.predict(X_train)
forest_mse = mean_squared_error(y_train, y_train_forest_pred)
forest_rmse = np.sqrt(forest_mse)
forest_rmse

"""
Explanation: It does worse on the test data as the CV score had told us. 
3. Random Forest Tree Regression
End of explanation
"""


scores_forest = cross_val_score(forest_reg, X_train, y_train,
                             scoring='neg_mean_squared_error', cv=10)
rmse_scores_forest = np.sqrt(-scores_forest)
print('mean:', rmse_scores_forest.mean(), 'and', 'std:', rmse_scores_forest.std())

"""
Explanation: 3.1 CV
End of explanation
"""


from sklearn.model_selection import GridSearchCV
param_grid = [
    {'n_estimators': [3, 10, 30], 'max_features': [2, 4, 6, 8]},
    {'bootstrap': [False], 'n_estimators': [3, 10], 'max_features': [2, 3, 4]}
]

forest_reg = RandomForestRegressor()

grid_search = GridSearchCV(forest_reg, param_grid, cv=5,
                          scoring ='neg_mean_squared_error')

grid_search.fit(X_train, y_train)

grid_search.best_params_

"""
Explanation: Forest tree also overfit on the training set.
We will have to regularize it.
3.2 Grid Search
Let's the best combination of hyperparameter values for the RandomForestRegressor. The grid search will explore 18 combinations of hyperparameter values and will train each model 5 times per cv.
End of explanation
"""


cvres = grid_search.cv_results_
for mean_score, params in zip(cvres['mean_test_score'], cvres['params']):
    print(np.sqrt(-mean_score), params)

feature_importances = grid_search.best_estimator_.feature_importances_
feature_importances

sorted(zip(feature_importances, X_train_df.columns), reverse=True)

"""
Explanation: 3.3 Reevaluating Features
End of explanation
"""


X_train_df_last = X_train_df.drop(['num_ans>= 29'], axis=1)

X_train_last = X_train_df_last.values

forest_reg.fit(X_train_last, y_train)
y_train_forest_pred_last = forest_reg.predict(X_train_last)
forest_mse_last = mean_squared_error(y_train, y_train_forest_pred_last)
forest_rmse_last = np.sqrt(forest_mse_last)
forest_rmse_last

scores_forest_last = cross_val_score(forest_reg, X_train_last, y_train,
                             scoring='neg_mean_squared_error', cv=10)
rmse_scores_forest_last = np.sqrt(-scores_forest_last)
print('mean:', rmse_scores_forest_last.mean(), 'and', 'std:', rmse_scores_forest_last.std())

"""
Explanation: It appears that num_ans&gt;=29 is not really important, while on the other hand, topics_followers is the most important feature of all. Let's see if dropping it will improve things.
End of explanation
"""


X_test_df_last = pd.DataFrame(X_test) #num>=29 is the fourth column

X_test_df_last.drop([3], axis=1, inplace=True)

X_test_last = X_test_df_last.values

forest_reg.fit(X_test_last, y_test)
y_test_forest_pred_last = forest_reg.predict(X_test_last)
forest_mse_last = mean_squared_error(y_test, y_test_forest_pred_last)
forest_rmse_last = np.sqrt(forest_mse_last)
forest_rmse_last

scores_forest_last = cross_val_score(forest_reg, X_test_last, y_test,
                             scoring='neg_mean_squared_error', cv=10)
rmse_scores_forest_last = np.sqrt(-scores_forest_last)
print('mean:', rmse_scores_forest_last.mean(), 'and', 'std:', rmse_scores_forest_last.std())

"""
Explanation: Sure, it improved by 1 but not really a big difference. How about on the test set?
End of explanation
"""


