cat ./poi_names.txt

"""
Explanation: Finding POI's in the Enron Dataset
Understanding the Dataset and Question
Data Exploration
Dataset Contents
final_project_dataset.pkl
Financial data from the included enron61712insiderpay.pdf have been combined into a dictionary in the included final_project_dataset.pkl file. In the dictionary, the key is the person's name, and the value is another dictionary, which contains the names of all the features and their values for that person. The features in the data fall into three major types, namely financial features, email features and POI labels.
financial features:
```python
(all units are in US dollars)
[
 'salary',
 'deferral_payments',
 'total_payments',
 'loan_advances',
 'bonus',
 'restricted_stock_deferred',
 'deferred_income',
 'total_stock_value',
 'expenses',
 'exercised_stock_options',
 'other',
 'long_term_incentive',
 'restricted_stock',
 'director_fees'
]
```
email features:
```python
(units are generally number of emails messages that reference the ; notable exception is ‘email_address’, which is a text string)
[
 'to_messages',
 'email_address',
 'from_poi_to_this_person',
 'from_messages',
 'from_this_person_to_poi',
 'poi', # POI Label (boolean, represented as integer).
 'shared_receipt_with_poi'
] 
```
Persons of Interest
End of explanation
"""


enron_data = pickle.load(open("./final_project_dataset.pkl"))

enron_data.iteritems().next()

"""
Explanation: This file contains a list of 35 people who were a person of interest in the Enron scandal. A POI is defined as someone who was:
* indicted
* settled without admitting guilt
* testified in exchange for immunity
Summary of Dataset
End of explanation
"""


# Replace "Nan" with NaN
for columns in enron_data.itervalues():
    for k,v in columns.iteritems():
        if type(v) is str and v.lower() == "nan":
            columns[k] = np.nan

enron_df = pd.DataFrame.from_dict(enron_data, orient="index")
enron_df

"""
Explanation: Features in the dataset:
In the dataset there are 146 employees with 20 features and 'poi' label for each employee. To further summarize the features in the dataset, we will convert the dictionary to a pandas DataFrame.
End of explanation
"""


# Omit the TOTAL index
enron_df.drop('TOTAL', inplace=True)

"""
Explanation: There is a row with the name, "TOTAL". This row should be removed.
End of explanation
"""


enron_df.loc[:, ['salary',
                 'deferral_payments',
                 'total_payments',
                 'loan_advances',
                 'bonus',
                 'restricted_stock_deferred',
                 'deferred_income',]].describe()

enron_df.loc[:, ['total_stock_value',
                 'expenses',
                 'exercised_stock_options',
                 'other',
                 'long_term_incentive',
                 'restricted_stock',
                 'director_fees']].describe()

"""
Explanation: Financial Features
End of explanation
"""


enron_df.loc[:, ['to_messages',
                 'email_address',
                 'from_poi_to_this_person',
                 'from_messages',
                 'from_this_person_to_poi',
                 'shared_receipt_with_poi']].describe()

"""
Explanation: Email Features
End of explanation
"""


enron_poi = enron_df[enron_df['poi']==True]
print("Number of POI's: " + str(len(enron_poi)))
enron_poi

"""
Explanation: Persons of Interest
End of explanation
"""


enron_df.isnull().sum()

sum(enron_df.isnull().sum())

"""
Explanation: Number of NaN's:
End of explanation
"""


enron_df.fillna(0, inplace=True)

"""
Explanation: Data Exploration Findings

145 people
20 features
18 persons of interest
1352 NaN entries
director_fees, loan_advances, and restricted_stock_deferred are the top 3 columns with the most NaN entries

Since there were so many 'NaN' entries, I went back to the pdf that the data was derived from and noticed that entries with '-' were being interpreted as NaN. To fix this I will be replacing the 'NaN's with the value zero.
End of explanation
"""


# Drop email_address column
enron_df.drop('email_address', axis=1, inplace=True)

enron_df.drop("THE TRAVEL AGENCY IN THE PARK", inplace=True)
enron_df.drop("LOCKHART EUGENE E", inplace=True)

"""
Explanation: Outlier Investigation
One outlier that was removed earlier was the "Total" index, which was represented the total sums of each columns. There were two entries that were invalid data points. One was "THE TRAVEL AGENCY IN THE PARK", which can not be a person and definitely not a person of interest. The other invalid entry was "LOCKHART EUGENE E", which had NaN values for all the features.
End of explanation
"""


enron_df['from_poi_ratio'] = enron_df['from_poi_to_this_person'] / enron_df['from_messages']
enron_df.fillna(0, inplace=True)

"""
Explanation: Indexes Removed
* TOTAL
* THE TRAVEL AGENCY IN THE PARK
* LOCKHART EUGENE E
The 'email_address' column was also removed since it is irrelevant data.
Optimize Feature Selection/Engineering
Create new features
Do POI's receive more emails from other POI's compared to non POI's?
End of explanation
"""


enron_df['to_poi_ratio'] = enron_df['from_this_person_to_poi'] / enron_df['to_messages']
enron_df.fillna(0, inplace=True)

"""
Explanation: Do POI's write more emails to other POI's compared to non POI's?
End of explanation
"""


enron_df['bonus_ratio'] = enron_df['bonus'] / enron_df['salary']

enron_df[['poi','bonus_ratio']]

"""
Explanation: Do POI's have a bigger bonus to salary ratio?
End of explanation
"""


enron_df.fillna(0, inplace=True)

"""
Explanation: For NaN values, the labels are more POI than not so these values will be filled with 0, since there seems to be a weak correlation between POI's and a large bonus_ratio.
End of explanation
"""


# Separate labels and features
enron_df_labels = enron_df['poi']
enron_df_features = enron_df[enron_df.columns.difference(['poi'])]

"""
Explanation: Feature Engineering Conclusion:

Three features were created:
The ratio of (emails from poi's sent to the person) to (from received)
The ratio of (emails from this person sent to poi's) to (emails sent)
The ratio of bonus to salary
In the next step feature selection will be done with selectKBest, PCA, and inherently with the feature importance in various tree classifier algorithms. 

Classifier Pipelines
End of explanation
"""


pipeline = Pipeline([
        ('kbest', SelectKBest()),
        ('gnb', GaussianNB())])
folds = 100
cv = StratifiedShuffleSplit(enron_df_labels, n_iter= folds, random_state = 42, test_size=0.20)
parameters = {"kbest__k": [1, 2, 3, 5, 8, 13, 19], "kbest__score_func": [f_classif]}
clf = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1')
clf.fit(enron_df_features, enron_df_labels)

kbest = clf.best_estimator_.steps[0][1]
kbest.get_support()
features = sorted(zip(enron_df_features.columns, kbest.scores_, kbest.get_support()), key=lambda x: x[1])

my_list = [x[0] for x in features if x[2] == True]
my_list = ['poi'] + my_list
my_list

data = enron_df[my_list].transpose().to_dict()

dump_classifier_and_data(GaussianNB(), data, my_list)

tester.main()

"""
Explanation: Baseline Classifier: Gaussian Naive Bayes
End of explanation
"""


pipeline = Pipeline([
        ("scale", preprocessing.StandardScaler()),
        ('pca', PCA()),
        ('gnb', GaussianNB())])
folds = 100
cv = StratifiedShuffleSplit(enron_df_labels, n_iter= folds, random_state = 42, test_size=0.20)
parameters = {
    "pca__n_components": [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19],
}
clf = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1')
clf.fit(enron_df_features, enron_df_labels)

pca = clf.best_estimator_.steps[1][1]

pca = clf.best_estimator_.steps[1][1]
pca.n_components

pca_nb = Pipeline([
        ("scale", preprocessing.StandardScaler()),
        ('pca', PCA(n_components=pca.n_components)),
        ('gnb', GaussianNB())])

features_list = list(enron_df.columns)
features_list.remove('poi')
features_list = ['poi'] + features_list

dump_classifier_and_data(pca_nb, enron_df.transpose().to_dict(), features_list)

tester.main()

"""
Explanation: Using PCA instead of selectKBest:
End of explanation
"""


pipeline = Pipeline([
        ('kbest', SelectKBest()),
        ('dt', DecisionTreeClassifier())])
folds = 100
cv = StratifiedShuffleSplit(enron_df_labels, n_iter= folds, random_state = 42, test_size=0.20)
parameters = {"kbest__k": [1, 2, 3, 5, 8, 13, 19], 'dt__max_features': [None, 'auto', 'log2'],
              'dt__criterion': ['gini', 'entropy']}
clf = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1')
clf.fit(enron_df_features, enron_df_labels)

kbest = clf.best_estimator_.steps[0][1]
kbest.get_support()
features = sorted(zip(enron_df_features.columns, kbest.scores_, kbest.get_support()), key=lambda x: x[1])

my_list = [x[0] for x in features if x[2] == True]
my_list = ['poi'] + my_list
my_list

clf.best_estimator_.steps[1][1]

data = enron_df[my_list].transpose().to_dict()
dump_classifier_and_data(clf.best_estimator_.steps[1][1], data, my_list)

tester.main()

"""
Explanation: PCA in our case performs poorly when compared to selectKBest. This indicates that variance is needed in the dataset.
Next Classifier: Decision Tree
With SelectKBest Feature Selection
End of explanation
"""


pipeline = Pipeline([
        ('dt', DecisionTreeClassifier())])
folds = 100
cv = StratifiedShuffleSplit(enron_df_labels, n_iter= folds, random_state = 42, test_size=0.20)
parameters = {'dt__max_features': [1, 2, 3, 5, 8, 13, 19],
              'dt__criterion': ['gini', 'entropy']}
clf = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1')
clf.fit(enron_df_features, enron_df_labels)

data = enron_df.transpose().to_dict()
dump_classifier_and_data(clf.best_estimator_.steps[0][1], data, features_list)

tester.main()

"""
Explanation: Without SelectKBest Feature Selection
End of explanation
"""


pipeline = Pipeline([
        ("scale", preprocessing.StandardScaler()),
        ('pca', PCA()),
        ('dt', DecisionTreeClassifier())])
folds = 100
cv = StratifiedShuffleSplit(enron_df_labels, n_iter= folds, random_state = 42, test_size=0.20)
parameters = {"pca__n_components": [2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17, 18, 19],
              'dt__max_features': [None, 'auto', 'log2'],
              'dt__criterion': ['gini', 'entropy']}
clf = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1')
clf.fit(enron_df_features, enron_df_labels)

pca = clf.best_estimator_.steps[1][1]
pca.n_components

pca_dt = Pipeline([
        ("scale", preprocessing.StandardScaler()),
        ('pca', PCA(n_components=pca.n_components)),
        ('dt', clf.best_estimator_.steps[2][1])])

dump_classifier_and_data(pca_dt, enron_df.transpose().to_dict(), features_list)
tester.main()

"""
Explanation: With PCA
End of explanation
"""


