from feature_selector import FeatureSelector

import pandas as pd

"""
Explanation: Introduction: Testing Feature Selector
In this notebook we will test the feature selector using two additional datasets. We will try out many of the FeatureSelector methods on these standard machine learning sets to make sure that it is a minimum working product.
https://github.com/WillKoehrsen/feature-selector
End of explanation
"""


air_quality = pd.read_csv('data/AirQualityUCI.csv')
air_quality['Date'] = pd.to_datetime(air_quality['Date'])
air_quality['Date'] = (air_quality['Date'] - air_quality['Date'].min()).dt.total_seconds()
air_quality['Time'] = [int(x[:2]) for x in air_quality['Time']]
air_quality.head()

labels = air_quality['PT08.S5(O3)']
air_quality = air_quality.drop(columns = 'PT08.S5(O3)')

fs = FeatureSelector(data = air_quality, labels = labels)

fs.identify_all(selection_params = {'missing_threshold': 0.5, 'correlation_threshold': 0.7, 
                                    'task': 'regression', 'eval_metric': 'l2', 
                                     'cumulative_importance': 0.9})

fs.plot_collinear()

fs.plot_missing()

fs.plot_feature_importances(threshold = 0.9)

fs.data_all.head()

air_quality_removed = fs.remove(methods = 'all', keep_one_hot=False)

fs.ops

fs.plot_collinear(plot_all=True)

"""
Explanation: Air Quality Dataset
Regression machine learning task
From UCI repository: https://archive.ics.uci.edu/ml/datasets/Air+Quality#
End of explanation
"""


insurance = pd.read_csv('data/caravan-insurance-challenge.csv')
insurance = insurance[insurance['ORIGIN'] == 'train']
labels = insurance['CARAVAN']
insurance = insurance.drop(columns = ['ORIGIN', 'CARAVAN'])
insurance.head()

fs = FeatureSelector(data = insurance, labels = labels)

fs.identify_all(selection_params = {'missing_threshold': 0.8, 'correlation_threshold': 0.85, 
                                    'task': 'classification', 'eval_metric': 'auc', 
                                     'cumulative_importance': 0.8})


fs.plot_feature_importances(threshold=0.8)

fs.plot_collinear()

insurance_missing_zero = fs.remove(methods = ['missing', 'zero_importance'])

to_remove = fs.check_removal()
fs.feature_importances.head()

insurance_removed = fs.remove(methods = 'all', keep_one_hot=False)

"""
Explanation: Insurance Dataset
Classification machine learning problem.
Formatted data from Kaggle: https://www.kaggle.com/uciml/caravan-insurance-challenge/data
Originally from UCI machine learning repository: https://archive.ics.uci.edu/ml/datasets/Insurance+Company+Benchmark+%28COIL+2000%29
End of explanation
"""


