main.getConnection()
# this will also create main.engine variable

model_id = 'titanic_training'

try:
    main.addModelId('titanic_training','Training on titanic data','passengerid')
except:
    pass
main.getSessionId(model_id)
# this will also create main.session_id variable

"""
Explanation: 1. Get connection; define model_id, get session id
End of explanation
"""


df = pd.read_csv('../data/titanic.csv')

df.head()

"""
Explanation: 2. Load the data
Could be the data from postgres SQL as well -> you would use pd.read_sql
End of explanation
"""


summary = get.getSummary(df)

summary.head()

store.storeSummary(summary)

## as we can see, the data is actually stored
pull.pullSummary(main.session_id)

"""
Explanation: 3. Get and store the summary
End of explanation
"""


y = df['survived_int']
X = df[['sex','pclass','embarked','title','age','family']]
X.index = df["passengerid"].tolist()

def preprocess_features(X):
    
    # Initialize new output DataFrame
    output = pd.DataFrame(index = X.index)

    # Investigate each feature column for the data
    for col, col_data in X.iteritems():

        # If data type is categorical, convert to dummy variables
        if col_data.dtype == object:
            col_data = pd.get_dummies(col_data, prefix = col)  
        
        # Collect the revised columns
        output = output.join(col_data)    
    return output

X = preprocess_features(X)

random.seed(0)
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8,  random_state=0)

clf = RandomForestClassifier(random_state=0) # just a basic random forest model
clf.fit(X_train, y_train)
## predict on the test set:
probs = clf.predict_proba(X_test)
score=[probs[x][1] for x in range(len(probs)) ]

"""
Explanation: 4. Create features matrix, train and test set, build a model on the training set -> predict
End of explanation
"""


test_results = get.getTestResults(score, y_test)
test_results.head(10)

store.storeTestResults(test_results)

## as we can see, the data is actually stored
pull.pullTestResults(main.session_id).head()

roc = pull.pullROC(main.session_id)
liftchart = pull.pullLiftChart(main.session_id)

fg = plt.figure(figsize=(10,5))
adj = plt.subplots_adjust(hspace=0.4,wspace=0.2)

sp = plt.subplot(1,2,1)
l1 = plt.plot(roc.false_positive_rate, roc.true_positive_rate)
tl = plt.title("ROC curve")

sp = plt.subplot(1,2,2)
l1 = plt.plot(liftchart.population, liftchart.target_population)
tl = plt.title("Liftchart")

plt.show()

pull.pullAccuracy(main.session_id, 0.5, 'population')

pull.pullAccuracy(main.session_id, 0.5, 'probability')

pull.pullConfMatrix(main.session_id, 0.5, 'probability')

threshold_value = 0.5 
threshold_type = "population"
main.updateThreshold(model_id, threshold_value, threshold_type)

"""
Explanation: 5. getTestResults; storeTestResults; pull some statistics
End of explanation
"""


store.storeModelScores(X_test.index, score)

## as we can see, the data is actually stored
pull.pullModelScores(main.session_id).head()

"""
Explanation: 6. Store the scores
End of explanation
"""


main.closeSession()

## as we can see, the end time is filled in
pd.read_sql("select * from model_factory.run_history where session_id='"+main.session_id+"'", main.engine)

"""
Explanation: 7. Close the session
End of explanation
"""


