import os
import pandas as pd
import seaborn as sns
import numpy as np
from scipy import stats, integrate
import matplotlib.pyplot as plt
import sklearn
from sklearn.model_selection import train_test_split
from sklearn import linear_model

os.getcwd()

a = pd.read_csv('per_scholas_data.csv', skiprows=range(1,2))
b = pd.read_csv('per_scholas_data2.csv')
c = pd.read_csv('per_scholas_data3.csv', skiprows=range(1,2))

a.head()

b.head()

c.head()

a.describe()

b.describe()

c.describe()

a.shape

b.shape

c.shape

merged_df = a.merge(b,on='Record ID').merge(c,on='Record ID')

merged_df.head()

merged_df.describe()

merged_df

"""
Explanation: Notebook for exporatory data analysis of Per Scholas Data
End of explanation
"""


labels = a.columns.values

labels

sum(a['Placed']) 

a_num = a[['First Post Training Wage', 'Retained (Months)', 'Current Wage']]

a_num_na = a_num.fillna(0)

first_wage = a_num_na['First Post Training Wage']

retained = a_num_na['Retained (Months)']

current_wage = a_num_na['Current Wage']

sns.jointplot(x=first_wage, y=current_wage)
plt.show()

"""
Explanation: Merged dataframe only containes 9 records...may not be very useful.  Go back to the first data frame.
End of explanation
"""


sns.jointplot(x=first_wage, y=retained, kind="kde");
plt.show()

"""
Explanation: There seems to be a good correlation between first post training wage and current wage
End of explanation
"""


x = first_wage
y = current_wage

X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)

lr = linear_model.LinearRegression()

X_train, X_test, y_train, y_test = X_train.reshape(-1,1), X_test.reshape(-1, 1), y_train.reshape(-1, 1), y_test.reshape(-1, 1)

y_train.shape

lr.fit(X_train, y_train)

lr.score(X_test, y_test)

plt.scatter(X_test, y_test,  color='black')
plt.plot(X_test, lr.predict(X_test), color='blue',
         linewidth=3)

plt.xticks(())
plt.yticks(())

plt.show()


"""
Explanation: Not much of correlation between first wage and retained duration
Let's go back to initial and current wage and see if we can find a relationship
End of explanation
"""


