!pip install feature_engine
!pip install arcticdata=="1.4"
"""
<br/><br/><br/><br/><br/>
![](https://i.imgur.com/9GaTOTy.png)
"""
"""
#### Hello! I'm Javier and im a Data Science Master's Degree student. I have made a library to automate the processes in which data scientists invest most part of the time. The library is divided into three sections: EDA, feature engineering and feature selection. 

### Give me a like if you found it useful and leave me a comment if you have any suggestion!
"""
"""
<br/><br/><br/><br/><br/>
![](https://i.imgur.com/HEmI1A5.png)
"""
import pandas as pd
import arcticdata.ExploratoryDataAnalysis as eda
import arcticdata.FeatureEngineering as fe
import arcticdata.FeatureSelection as fs

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))
"""
## Before starting with the dataset, the library counts with a project checklist if you are lost in any moment and you don't remember what you need to do (This checklist is based in Aurélien Géron book called "Hands-On Machine Learning with Scikit-Learn, Keras & Tensorflow")
"""
eda.project_checklist()
"""
<br/><br/><br/><br/><br/>
![](https://i.imgur.com/N16UsDR.png)
"""
"""
## To show what the library can do, we will work with a classic in Data Science, Titanic's dataset
"""
train = pd.read_csv('/kaggle/input/titanic/train.csv')
test = pd.read_csv('/kaggle/input/titanic/test.csv')

train.head(3)
"""
<br/><br/><br/><br/><br/>
![](https://i.imgur.com/wZjLyh8.png)
"""
"""
<br><br>
# Looking for correlations
"""
eda.corr_plot(train)
"""
<br><br>
# Looking for correlations for one feature
"""
eda.correlation(train,'SibSp')
"""
<br><br>
# Looking for missing values
"""
eda.missing_data(train,count=True)
eda.missing_data(train, percentage=True)
"""
<br><br>
# Distribution of numerical features
"""
eda.distribution_diag(train,'Age')
"""
<br><br>
# Outliers
"""
"""
### It's normal to have some outliers, but we need to easy detect if there is something wrong with our dataset. We can use IQR method (1.5*IQR) or IQR3 (3*IQR) to get the outliers
"""
eda.outliers(train,'Age',method='IQR',table=True, limits=True)
eda.outliers(train,'Age',method='IQR3',table=True, limits=True)
"""
<br><br>
# Value count for categorical features
"""
eda.value_counts(train.Embarked, count=True)
eda.value_counts(train.Embarked, percentage=True)
"""
<br><br>
# Getting all the functions together
"""
eda.feature_analysis(train, "Age", test=test)
eda.feature_analysis(train, "Embarked", test=test)
eda.feature_analysis(train, "Fare", test=test)
eda.feature_analysis(train, "Cabin", test=test)
"""
<br><br><br><br><br><br>
![](https://i.imgur.com/HCEMQgK.png)
"""
"""
# Imputing missing values
"""
eda.distribution_diag(train,'Age')
x=fe.impute_missing(train,train,method='median')
eda.distribution_diag(x,'Age')
x=fe.impute_missing(train,train,method='mean')
eda.distribution_diag(x,'Age')
x=fe.impute_missing(train,train,method='arbitrary',arbitrary_number=120)
eda.distribution_diag(x,'Age')
x=fe.impute_missing(train,train,method='tail',tail='right')
eda.distribution_diag(x,'Age')
x=fe.impute_missing(train,train,method='indicator')
x['Embarked_na'].sample(5)
x=fe.impute_missing(train,train,method='random')
eda.distribution_diag(x,'Age')
x=fe.impute_missing(train.Embarked.to_frame(),train.Embarked.to_frame(),method='mode')
eda.value_counts(x.Embarked,percentage=True)
x=fe.impute_missing(train.Embarked.to_frame(),train.Embarked.to_frame(),method='category')
eda.value_counts(x.Embarked,percentage=True)
train = fe.impute_missing(train,train,method='random')
test = fe.impute_missing(train.drop("Survived",axis=1),test,method='random')
"""
<br><br>
# Grouping rare labels
"""
train['Cabin'] = train['Cabin'].str[0]
test['Cabin'] = test['Cabin'].str[0]
eda.value_counts(train.Cabin,percentage=True)
x=fe.group_rare_labels(train.Cabin,train.Cabin,tol=0.05)
eda.value_counts(x.Cabin,percentage=True)
train.drop("Name",axis=1,inplace=True)
train.drop("Ticket",axis=1,inplace=True)
test.drop("Name",axis=1,inplace=True)
test.drop("Ticket",axis=1,inplace=True)
"""
<br><br>
# Transformation of numerical features
"""
eda.distribution_diag(train,'Age')
x = fe.transform(train,'log',variables=['Age'])
eda.distribution_diag(x,'Age')
x = fe.transform(train,'exp',variables=['Age'])
eda.distribution_diag(x,'Age')
x = fe.transform(train,'inv',variables=['Age'])
eda.distribution_diag(x,'Age')
x = fe.transform(train,'boxcox',variables=['Age'])
eda.distribution_diag(x,'Age')
x = fe.transform(train,'yeo',variables=['Age'])
eda.distribution_diag(x,'Age')
"""
<br><br>
# Discretization of numerical features
"""
x = fe.discretization(train, train, method='equalfreq',variables=['Age'])
eda.value_counts(x.Age,percentage=True,table=False)
x = fe.discretization(train, train, method='equalrange',variables=['Age'])
eda.value_counts(x.Age,percentage=True,table=False)
x = fe.discretization(train, train, method='equalfreq',variables=['Age'],return_boundaries=True)
eda.value_counts(x.Age,percentage=True,table=False)
"""
<br><br>
# Encoding of categorical features
"""
x = fe.encoding(train, train, method='order', variables=['Cabin'])
x.head()
x = fe.encoding(train, train, method='onehot', variables=['Cabin'])
x.head()
test = fe.encoding(train.drop("Survived",axis=1), test)
train = fe.encoding(train, train)
"""
<br><br>
# Numerical features scaling
"""
x = fe.scale(train, method='standar')
x.head()
x = fe.scale(train, method='normal')
x.head()
x = fe.scale(train, method='minmax')
x.head()
train = fe.scale(train)
test = fe.scale(test)
"""
<br><br><br><br><br><br>
![](https://i.imgur.com/TdKX2gW.png)
"""
"""
# Drop constant features
"""
x = fs.dropconstant(train, tol=0.998)
x.head()
"""
<br><br>
# Drop duplicated features
"""
x = fs.dropduplicated(train)
x.head()
"""
<br><br>
# Drop correlated features
"""
x = fs.dropcorrelated(train, threshold=0.8)
x.head()
"""
<br><br>
# Step forward feature selection
"""
from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier()
fs.step_forward(train.drop("Survived",axis=1), train['Survived'],model,kfeatures=16,info=True)
"""
<br><br>
# Thanks for watching! Like the notebook if you like it!
"""