import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

plt.style.use('ggplot')
plt.rcParams['figure.figsize'] = (12, 8)

# Load the train dataset
train = pd.read_csv("../Data/train.csv")

# Load the test dataset
test = pd.read_csv("../Data/test.csv")

# View the first 5 records of train
train.head()

# View the last 10 records of test
test.tail(10)

# List the attributes/feature names/columns in train dataset
train.columns

# List the attributes in test dataset. 
test.columns

type(test.columns)

train.columns.values

test.columns.values

# Do they match with train?
[x in test.columns.values for x in train.columns.values]

"""
Explanation: 1. Frame
A Portuguese Bank wants to run a direct marketing campaign to sell its new term deposit plan. The goal is to help them identify customers who would most likely buy the plan?
Open Discussion How to approach this problem?
2. Acquire
UCI has a number of datasets related to machine learning. We will leverage the Bank Marketing dataset.
Look into this link for more information
https://archive.ics.uci.edu/ml/datasets/Bank+Marketing
Load the train and test datasets
End of explanation
"""


train.dtypes

# Find unique values in deposit for train dataset
pd.unique(train.deposit)

# Find unique values in deposit for test dataset. Are they the same? 
pd.unique(test.deposit)

pd.unique(test['month'])

# Find frequency of deposit in train dataset
train.deposit.value_counts()

# Find frequency of deposit in test dataset
test.deposit.value_counts()

type(train.deposit.value_counts())

# Is the distribution of deposit similar in train and test?
print("train:",train.deposit.value_counts()[1]/train.shape[0]*100)
print("test:",test.deposit.value_counts()[1]/test.shape[0]*100)

# Find number of rows and columns in train 
train.shape

# Find number of rows and columns in test
test.shape

"""
Explanation: Attribute Information:
Input variables:
bank client data:

age (numeric)
job : type of job (categorical: 'admin.','blue-collar','entrepreneur','housemaid','management','retired','self-employed','services','student','technician','unemployed','unknown')
marital : marital status (categorical: 'divorced','married','single','unknown'; note: 'divorced' means divorced or widowed)
education (categorical: 'basic.4y','basic.6y','basic.9y','high.school','illiterate','professional.course','university.degree','unknown')
default: has credit in default? (categorical: 'no','yes','unknown')
housing: has housing loan? (categorical: 'no','yes','unknown')
loan: has personal loan? (categorical: 'no','yes','unknown')

related with the last contact of the current campaign:

contact: contact communication type (categorical: 'cellular','telephone') 
month: last contact month of year (categorical: 'jan', 'feb', 'mar', ..., 'nov', 'dec')
day_of_week: last contact day of the week (categorical: 'mon','tue','wed','thu','fri')
duration: last contact duration, in seconds (numeric). Important note: this attribute highly affects the output target (e.g., if duration=0 then y='no'). Yet, the duration is not known before a call is performed. Also, after the end of the call y is obviously known. Thus, this input should only be included for benchmark purposes and should be discarded if the intention is to have a realistic predictive model.

other attributes:

campaign: number of contacts performed during this campaign and for this client (numeric, includes last contact)
pdays: number of days that passed by after the client was last contacted from a previous campaign (numeric; 999 means client was not previously contacted)
previous: number of contacts performed before this campaign and for this client (numeric)
poutcome: outcome of the previous marketing campaign (categorical: 'failure','nonexistent','success')

social and economic context attributes

emp.var.rate: employment variation rate - quarterly indicator (numeric)
cons.price.idx: consumer price index - monthly indicator (numeric) 
cons.conf.idx: consumer confidence index - monthly indicator (numeric) 
euribor3m: euribor 3 month rate - daily indicator (numeric)
nr.employed: number of employees - quarterly indicator (numeric)

Output variable (desired target):
y - has the client subscribed a term deposit? (binary: 'yes','no')
3. Explore
End of explanation
"""


train.describe()

"""
Explanation: Find basic summary metrics for the train dataframe
End of explanation
"""


# Create labels: that has 0 for No and 1 for Yes in train dataset
labels = np.where(train.deposit=="no", 0, 1)

# Display number of 0 and 1 - check if it is the same as what we saw above?
np.unique(labels, return_counts=True)

"""
Explanation: *Where did the remaining columns go? *
describe only works for continuous variable and not categorical variables
Plots
End of explanation
"""


train.head()

train.loc[:,['deposit','age']]

bivariate_plot_deposit_age = train.loc[:,["deposit", "age"]].copy()

bivariate_plot_deposit_age.head()

bivariate_plot_deposit_age.age.hist()

sns.stripplot(x="deposit", y = "age", data = bivariate_plot_deposit_age,
             jitter = True, alpha = 0.1)

"""
Explanation: Bi-variate plot : Deposit vs age
End of explanation
"""


train.plot(kind="scatter", x = 'age', y = 'pdays', color = labels, alpha = 0.5, s=50)

"""
Explanation: Multivariate plot : Deposit vs age and pdays
End of explanation
"""


train.plot(kind="scatter", x = 'day', y = 'duration', color = labels, 
           alpha = 0.5, s=50)

"""
Explanation: Multivariate plot : Deposit vs day and duration
End of explanation
"""


import sklearn
from sklearn import preprocessing

# Find the columns that are categorical
train.select_dtypes(include=['object'])

train_to_convert = train.select_dtypes(include=["object_"]).copy()
test_to_convert = test.select_dtypes(include=["object_"]).copy()

train_np = np.array(train_to_convert)
test_np = np.array(test_to_convert)

for i in range(train_np.shape[1]):
    lbl = preprocessing.LabelEncoder()
    lbl.fit(list(train_np[:, i]))
    train_np[:,i] = lbl.transform(train_np[:, i])
    test_np[:,i] = lbl.transform(test_np[:, i])
    

# Display train_np
train_np

# How would you transform test?
test_np

train_np

test_np

# Now, merge the numeric and encoded train variables into one single dataset

train_numeric = np.array(train.select_dtypes(exclude=["object_"]).copy())

train_numeric.shape

train_encoded = np.concatenate([train_numeric, train_np], axis=1)

# Now, merge the numeric and encoded test variables into one single dataset
test_numeric = np.array(test.select_dtypes(exclude=["object_"]).copy())

test_encoded = np.concatenate([test_numeric, test_np], axis=1)

"""
Explanation: 4. Refine
covert categorical variables to numeric
Two options:
1. Label Encoder
2. One-Hot Encoding
Label Encoding
End of explanation
"""


# Create train X and train Y

xlen = train_encoded.shape[1]-1

train_encoded_X = train_encoded[:, :xlen]

train_encoded_Y = np.array(train_encoded[:, -1], dtype=float)

train_encoded_Y

# Create test X
test_encoded_X = test_encoded[:, :xlen]

# Create test Y
test_encoded_Y = np.array(test_encoded[:, -1], dtype=float)

"""
Explanation: 5. Model
End of explanation
"""


model_allzero = test_encoded_Y.copy()

model_allzero = 0

# The mean square error on AllZero model
print("Mean Squared Error on all zero model: %.2f"
      % (np.mean((model_allzero - test_encoded_Y) ** 2)*100))

"""
Explanation: Benchmark Model
End of explanation
"""


from sklearn import linear_model

model_linear = linear_model.LinearRegression()

model_linear.fit(train_encoded_X, train_encoded_Y)

# The coefficients
print('Coefficients: \n', model_linear.coef_)

# Prediction
model_linear_prediction = model_linear.predict(test_encoded_X)

model_linear_prediction

model_linear_prediction = np.where(model_linear_prediction>0.5, 1, 0)

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_linear.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_linear.predict(test_encoded_X) - test_encoded_Y) ** 2)*100))

"""
Explanation: First Model: Linear Regression
Y = β0 + β1X1 + β2X2 + … + βn*Xn
End of explanation
"""


model_logistic_L2 = linear_model.LogisticRegression()

model_logistic_L2.fit(train_encoded_X, train_encoded_Y)

# The coefficients
print('Coefficients: \n', model_logistic_L2.coef_)

# Prediction
model_logistic_L2_prediction = model_logistic_L2.predict(test_encoded_X)

np.unique(model_logistic_L2_prediction)

np.sum(model_logistic_L2_prediction)

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_logistic_L2.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_logistic_L2_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Second Model: L2 Logistic Regression
End of explanation
"""


# Code here. Report evaulation
model_logistic_L1 = linear_model.LogisticRegression(penalty = 'l1')

model_logistic_L1.fit(train_encoded_X, train_encoded_Y)

# The coefficients
print('Coefficients: \n', model_logistic_L1.coef_)

# Prediction
model_logistic_L1_prediction = model_logistic_L1.predict(test_encoded_X)

np.unique(model_logistic_L1_prediction)

np.sum(model_logistic_L1_prediction)

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_logistic_L1.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_logistic_L1_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Third Model: L1 Logistic Regression
End of explanation
"""


model_logistic_L2C = linear_model.LogisticRegression(C = 2)

model_logistic_L2C.fit(train_encoded_X, train_encoded_Y)

# The coefficients
print('Coefficients: \n', model_logistic_L2C.coef_)

# Prediction
model_logistic_L2C_prediction = model_logistic_L2C.predict(test_encoded_X)

np.unique(model_logistic_L2C_prediction)

np.sum(model_logistic_L2C_prediction)

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_logistic_L2C.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_logistic_L2C_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Fourth Model: L1 Logistic Regression - Change value of C
End of explanation
"""


from sklearn import tree
from sklearn.externals.six import StringIO
# import pydot

model_DT = tree.DecisionTreeClassifier()

#Let's use only the first two columns as features for the model

model_DT.fit(train_encoded_X[:,1:3], train_encoded_Y)

# dot_data = StringIO() 
# tree.export_graphviz(model_DT, out_file=dot_data) 
# graph = pydot.graph_from_dot_data(dot_data.getvalue()) 
# graph.write_pdf("dt1.pdf") 

# Prediction
model_DT_prediction = model_DT.predict(test_encoded_X[:,1:3])

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_DT.predict(train_encoded_X[:,1:3]) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_DT_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Fifth Model: Decision  Tree Model
End of explanation
"""


model_DTAll = tree.DecisionTreeClassifier()

model_DTAll.fit(train_encoded_X, train_encoded_Y)

# Prediction
model_DTAll_prediction = model_DTAll.predict(test_encoded_X)

# The mean square error on train
print("Mean Squared Error on train: %.2f"
      % (np.mean((model_DTAll.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Squared Error on test: %.2f"
      % (np.mean((model_DTAll_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Decision Tree is prone to overfitting !
Now, use all the features and build the model. Report the accuracy
End of explanation
"""


from sklearn.ensemble import RandomForestClassifier

?RandomForestClassifier

model_RF = RandomForestClassifier()

model_RF.fit(train_encoded_X, train_encoded_Y)

# Prediction
model_RF_prediction = model_RF.predict(test_encoded_X)

# The mean square error on train
print("Mean Percentage Error on train: %.2f"
      % (np.mean((model_RF.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Percentage Error on test: %.2f"
      % (np.mean((model_RF_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Sixth Model: Random Forest Model
End of explanation
"""


?RandomForestClassifier

model_RFMod = RandomForestClassifier(max_depth = 8, oob_score = True, n_estimators = 400 )

model_RFMod.fit(train_encoded_X, train_encoded_Y)

# Prediction
model_RFMod_prediction = model_RFMod.predict(test_encoded_X)

# The mean square error on train
print("Mean Percentage Error on train: %.2f"
      % (np.mean((model_RFMod.predict(train_encoded_X) - train_encoded_Y) ** 2)*100))

# The mean square error on test
print("Mean Percentage Error on test: %.2f"
      % (np.mean((model_RFMod_prediction - test_encoded_Y) ** 2)*100))

"""
Explanation: Let's change model parameters

Use 400 trees
use maximum depth of 8
print Out of Bag (OOB) score
End of explanation
"""


from sklearn.cross_validation import StratifiedKFold

?StratifiedKFold

skf = StratifiedKFold(train_encoded_Y, 5, random_state=1131, shuffle=True)

for train, test in skf:
    print("%s %s" % (train, test))
    print(train.shape, test.shape)

model_RF = RandomForestClassifier()

for k, (train, test) in enumerate(skf):
    model_RF.fit(train_encoded_X[train], train_encoded_Y[train])
    print("fold:", k+1, model_RF.score(train_encoded_X[test], train_encoded_Y[test]))
    

"""
Explanation: Cross Validation
End of explanation
"""


cv_error = []
for k, (train, test) in enumerate(skf):
    model_RF.fit(train_encoded_X[train], train_encoded_Y[train])
    print(k)
    cv_error.append(np.mean((model_RFMod.predict(train_encoded_X) - train_encoded_Y) ** 2)*100)

cv_error

# code here
np.mean(cv_error)

"""
Explanation: Find mean CV error
End of explanation
"""


train = pd.read_csv("../Data/train.csv")
test = pd.read_csv("../Data/test.csv")

train_one_hot = pd.get_dummies(train)

train_one_hot.head()

test_one_hot = pd.get_dummies(test)
test_one_hot.head()

# Check if columns are the same

[x in test_one_hot.columns.values for x in train_one_hot.columns.values]

#Create train X , train Y, test X , test Y

train_X = train_one_hot.ix[:,:train_one_hot.shape[1]-2 ]

train_X.columns

train_Y = train_one_hot.ix[:, -1]

train_Y.head()

test_X = test_one_hot.ix[:,:test_one_hot.shape[1]-2 ]
test_Y = test_one_hot.ix[:, -1]

# Run Random Forest and check accuracy

model_RF = RandomForestClassifier(n_estimators=400, max_depth=8, oob_score=True, n_jobs=-1)

model_RF.fit(train_X, train_Y)

# Prediction
model_RF_prediction = model_RF.predict(test_X)

# The mean square error on train
print("Mean Percentage Error on train: %.2f"
      % (np.mean((model_RF.predict(train_X) - train_Y) ** 2)*100))

# The mean square error on test
print("Mean Percentage Error on test: %.2f"
      % (np.mean((model_RF_prediction - test_Y) ** 2)*100))

"""
Explanation: Run for different parameters, different models and find mean CV error and for different KFolds
One Hot Encoding
End of explanation
"""


