# loading libraries
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
from sklearn.tree import DecisionTreeClassifier
import seaborn as sns
# loading dataset and checking the shape and data head
breast_cancer = pd.read_csv('../input/breast-cancer-wisconsin-data/data.csv')
print("shape of the data: ", breast_cancer.shape)
breast_cancer.head()
# checking the data info 
breast_cancer.info()
# dropping the Unnamed: 32 column since all the values are empty
breast_cancer = breast_cancer.drop(['Unnamed: 32'], axis=1)

# checking sum of null values 
breast_cancer.isnull().sum()
# encoding categorical variables on diagnosis column
print("unique values", breast_cancer['diagnosis'].unique())
print("before encoding -->")
print(breast_cancer['diagnosis'].tail())

enc = LabelEncoder()
breast_cancer['diagnosis'] = enc.fit_transform(breast_cancer['diagnosis'])

print("after encoding -->")
print(breast_cancer['diagnosis'].tail())
# checking feature correlation
b_corr = breast_cancer.corr()
b_corr

plt.figure(figsize=(25,10))
sns.heatmap(b_corr, cmap = sns.color_palette("vlag", as_cmap=True), annot=True)
# dropping features based on correlation
breast_cancer = breast_cancer.drop(['id', 'symmetry_mean', 'concavity_se', 'compactness_se' ], axis=1)
# Splitting the dataset into features and labels
# diagnosis column indicates the labels
features = breast_cancer.drop(['diagnosis'], axis=1)
labels = breast_cancer[['diagnosis']]
print("features shape", features.shape)
print("label shape", labels.shape)

# splitting test and train data
X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.3, random_state=42)
print("training data shape", X_train.shape)
print("testing data shape", X_test.shape)
# scaling all the values between 0-1
scaler = MinMaxScaler(copy=False)
scaler.fit(X_train)
scaler.transform(X_train)
scaler.transform(X_test)

# data head after scaling
print("minimum value", X_train.min(axis=0).values)
print("maximum value", X_train.max(axis=0).values)
X_train.head()
# logistic regression model
model = LogisticRegression()
model.fit(X_train, y_train.values.ravel())
y_pred = model.predict(X_test)
lg_accuracy = accuracy_score(y_test, y_pred)
print("Accuracy using logistic regression: ", lg_accuracy)
# Decision tree model
clf = DecisionTreeClassifier(criterion='entropy', random_state=42)
clf.fit(X_train,y_train)
y_pred = clf.predict(X_test)
dt_accuracy = accuracy_score(y_test, y_pred)
print("Accuracy using decision tree: ", dt_accuracy)
# plotting bar chart
sns.set_style("whitegrid")
sns.set(rc={'figure.figsize':(6,6)})

ax = sns.barplot(x=['Logistic regression', 'Decision tree'], y=[lg_accuracy, dt_accuracy])
ax.set(xlabel='Models', ylabel='Accuracy', title='Classification Model Comparison')

a = int(lg_accuracy * 100)
b = int(dt_accuracy * 100)
ax.text(0, lg_accuracy, str(a)+'%')
ax.text(1, dt_accuracy, str(b)+'%')
plt.show()
from sklearn import svm
svm_clf = svm.SVC()

svm_clf.fit(X_train,y_train.values.ravel())

print("Training accuracy of the model is", (svm_clf.score(X_train, y_train)))
svm_accuracy = svm_clf.score(X_test, y_test)
print("Testing accuracy of the model is", svm_accuracy)
from sklearn.neural_network import MLPClassifier
mlp_clf = MLPClassifier(hidden_layer_sizes=(7), activation="relu", random_state=1, max_iter=10000)

mlp_clf.fit(X_train, y_train.values.ravel())

print("Training accuracy of the model is", (mlp_clf.score(X_train, y_train)))
nn_accuracy = mlp_clf.score(X_test, y_test)
print("Testing accuracy of the model is", nn_accuracy)
from sklearn.ensemble import RandomForestClassifier
rf_clf = RandomForestClassifier(n_estimators=50)

rf_clf.fit(X_train, y_train.values.ravel())
                            
print("Training accuracy of the model is", (rf_clf.score(X_train, y_train)))
rf_accuracy = rf_clf.score(X_test, y_test)
print("Testing accuracy of the model is", rf_accuracy)                             
# plotting bar chart
sns.set_style("whitegrid")
sns.set(rc={'figure.figsize':(6,6)})

ax = sns.barplot(x=['svm', 'nn', 'rf'], y=[svm_accuracy, nn_accuracy, rf_accuracy])
ax.set(xlabel='Models', ylabel='Accuracy', title='Classification Model Comparison')

a = int(svm_accuracy * 100)
b = int(nn_accuracy * 100)
c = int(rf_accuracy * 100)
ax.text(0, svm_accuracy, str(a)+'%')
ax.text(1, nn_accuracy, str(b)+'%')
ax.text(2, rf_accuracy, str(c)+'%')
plt.show()
# Performing dimensionality reduction using PCA
from sklearn.decomposition import PCA
pca = PCA(n_components=15)
pca.fit(X_train)
X_train_pca = pca.fit_transform(X_train)
X_test_pca = pca.fit_transform(X_test)

print(X_train_pca)
sum(pca.explained_variance_ratio_)
from sklearn import svm
svm_model = svm.SVC()

svm_model.fit(X_train_pca, y_train.values.ravel())

print("Training accuracy of the model is", (svm_model.score(X_train_pca, y_train)))
pca_svm_accuracy = svm_model.score(X_test_pca, y_test)
print("Testing accuracy of the model is", pca_svm_accuracy)
from sklearn.neural_network import MLPClassifier
mlpc_model = MLPClassifier(hidden_layer_sizes=(7), activation="relu", random_state=1, max_iter=10000)

mlpc_model.fit(X_train_pca, y_train.values.ravel())

print("Training accuracy of the model is", (mlpc_model.score(X_train_pca, y_train)))
pca_nn_accuracy = mlpc_model.score(X_test_pca, y_test)
print("Testing accuracy of the model is", pca_nn_accuracy)
from sklearn.ensemble import RandomForestClassifier
rf_model = RandomForestClassifier(n_estimators=50)

rf_model.fit(X_train_pca, y_train.values.ravel())
                            
print("Training accuracy of the model is", (rf_model.score(X_train_pca, y_train)))
pca_rf_accuracy = rf_model.score(X_test_pca, y_test)
print("Testing accuracy of the model is", pca_rf_accuracy)                             
# plotting bar chart for models with pca
sns.set_style("whitegrid")
sns.set(rc={'figure.figsize':(6,6)})

ax = sns.barplot(x=['svm_pca', 'nn_pca', 'rf_pca'], y=[pca_svm_accuracy, pca_nn_accuracy, pca_rf_accuracy])
ax.set(xlabel='Models', ylabel='Accuracy', title='Classification Model Comparison')

a = int(pca_svm_accuracy * 100)
b = int(pca_nn_accuracy * 100)
c = int(pca_rf_accuracy * 100)
ax.text(0, pca_svm_accuracy, str(a)+'%')
ax.text(1, pca_nn_accuracy, str(b)+'%')
ax.text(2, pca_rf_accuracy, str(c)+'%')
plt.show()
# plotting bar chart
sns.set_style("whitegrid")
sns.set(rc={'figure.figsize':(10,10)})

ax = sns.barplot(x=['SVM', 'NN', 'RF', 'SVM(with pca)', 'NN(with pca)', 'RF(with pca)'], y=[svm_accuracy, nn_accuracy, rf_accuracy, pca_svm_accuracy, pca_nn_accuracy, pca_rf_accuracy])
ax.set(xlabel='Models', ylabel='Accuracy', title='Classification Model Comparison')

a = int(svm_accuracy * 100)
b = int(nn_accuracy * 100)
c = int(rf_accuracy * 100)
x = int(pca_svm_accuracy * 100)
y = int(pca_nn_accuracy * 100)
z = int(pca_rf_accuracy * 100)
ax.text(0, svm_accuracy, str(a)+'%')
ax.text(1, nn_accuracy, str(b)+'%')
ax.text(2, rf_accuracy, str(c)+'%')
ax.text(3, pca_svm_accuracy, str(x)+'%')
ax.text(4, pca_nn_accuracy, str(y)+'%')
ax.text(5, pca_rf_accuracy, str(z)+'%')

plt.show()
"""
ways to find the most representative features:
"""
#Univariate feature selection
#Selected 20 features according to the k highest scores and using knn
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from sklearn.neighbors import KNeighborsClassifier

select_feature = SelectKBest(chi2, k=20).fit(X_train, y_train)

x_train_2 = select_feature.transform(X_train)
x_test_2 = select_feature.transform(X_test)
print('Score list:', select_feature.scores_)
print('Feature list:', X_train.columns)

knn=KNeighborsClassifier()
print("for Selected features")
knn.fit(x_train_2, y_train.values.ravel())
print("Test set accuracy: {:.2f}".format(knn.score(x_test_2, y_test.values.ravel())))
print("for all features")
knn.fit(X_train, y_train.values.ravel())
print("Test set accuracy: {:.2f}".format(knn.score(X_test, y_test.values.ravel())))
#Recursive feature elimination (RFE)
from sklearn.feature_selection import RFE
from sklearn.ensemble import RandomForestClassifier
# Create the RFE object and rank each pixel
clf_rf_3 = RandomForestClassifier()      
rfe = RFE(estimator=clf_rf_3, n_features_to_select=10
          , step=1)
rfe = rfe.fit(X_train, y_train.values.ravel())
print('Chosen best features by rfe:',X_train.columns[rfe.support_])
#Recursive feature elimination with cross validation
#finding optimal number of features needed for best accuracy
# The "accuracy" scoring is proportional to the number of correct classifications
from sklearn.feature_selection import RFECV
from sklearn.ensemble import RandomForestClassifier
clf_rf_4 = RandomForestClassifier() 
rfecv = RFECV(estimator=clf_rf_4, step=1, cv=5,scoring='accuracy')   #5-fold cross-validation
rfecv = rfecv.fit(X_train, y_train.values.ravel())

print('Optimal number of features :', rfecv.n_features_)
print('Best features :', X_train.columns[rfecv.support_])