# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the read-only "../input/" directory
# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory

import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
df=pd.read_csv("../input/breast-cancer-wisconsin-data/data.csv")
df
df["id"]
df.drop("id",axis=1, inplace=True)
df
df.drop("Unnamed: 32",axis=1, inplace=True)
df

df.columns
df.isnull().sum()
from sklearn.preprocessing import LabelEncoder
le=LabelEncoder()
le.fit_transform(df["diagnosis"])
df["diagnosis"]=le.fit_transform(df["diagnosis"])
df
df["diagnosis"].value_counts()
df.corr()
import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(15,10))
sns.heatmap(df.corr(),cmap="inferno")
df.columns
X=df.drop("diagnosis",axis=1)
X=X.values
X
y=df["diagnosis"]
y=y.values
y
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test=train_test_split(X,y, test_size=0.2)
X_train.shape
y_train.shape
X_test.shape
y_test.shape
from sklearn.preprocessing import StandardScaler
sc=StandardScaler()
X_train=sc.fit_transform(X_train)
X_train
X_test=sc.transform(X_test)
X_test
from tensorflow.keras.models import Sequential # creates`Sequential` groups a linear stack of layers into a `tf.keras.Model`.
from tensorflow.keras.layers import Dense #regularly connected neurons)
ann=Sequential()
ann.add(Dense(units=30, activation="relu")) # here we add a dense layer with 30 neurons
ann.add(Dense(units=15, activation="relu")) # here we add a dense layer with 15 neurons
"""
The activation function is a mathematical “gate” in between the input feeding the current neuron and its output going to the next layer. It can be as simple as a step function that turns the neuron output on and off, depending on a rule or threshold. Or it can be a transformation that maps the input signals into output signals that are needed for the neural network to function.


Types of Activation Function:

1.Sigmoid / Logistic

2.TanH / Hyperbolic Tangent

3.ReLU (Rectified Linear Unit)

4.Softmax
"""
ann.add(Dense(1,activation="sigmoid"))  # here we add output layer with 1 neuron
"""
 optimizer:this determines how we want to perform the gradient descent like adam optimizer loss= represents cost function we want to use

Type of Optimizers:

1.Gradient Descent

2.Stochastic Gradient Descent

3.Mini-Batch Gradient Descent

4.Adam:The intuition behind the Adam is that we don’t want to roll so fast just because we can jump over the minimum, we want to decrease the velocity a little bit for a careful search.


Choosing an optimizer: Keep in mind what kind of problem we are trying to solve:

For a multi-class classification problem:

model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])

For a binary classification problem:

model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['accuracy'])

For a mean squared error regression problem:

model.compile(optimizer='rmsprop', loss='mse')
"""
ann.compile(optimizer="adam", loss="binary_crossentropy",metrics=["accuracy"])
ann.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test))
pd.DataFrame(ann.history.history)
pd.DataFrame(ann.history.history).plot(figsize=(15,10))
ann.evaluate(X_train, y_train)
ann.evaluate(X_test, y_test)
X_test
X_test[[1]]
ann.predict(X_test[[1]])
y_test[[1]]
predictions=ann.predict(X_test)
predictions
predictions_df=pd.DataFrame(predictions, columns=["Predictions"])
predictions_df.head()
y_test_df=pd.DataFrame(y_test, columns=["Diagnosis"])
y_test_df.head()
comparison_df=pd.concat([predictions_df, y_test_df], axis=1)
comparison_df.head()
ann2=Sequential()
ann2.add(Dense(units=1, activation="relu")) # here we add a dense layer with 1 neuron
ann2.add(Dense(1,activation="sigmoid")) 
ann2.compile(optimizer="adam", loss="binary_crossentropy",metrics=["accuracy"])
ann2.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test))
ann2.evaluate(X_train, y_train)
ann2.evaluate(X_test, y_test)
ann3=Sequential()
"""
In order to see the overfitting:
"""
ann3.add(Dense(units=100, activation="relu")) # here we add a dense layer with 100 neurons
ann3.add(Dense(units=100, activation="relu")) # here we add a dense layer with 100 neurons
ann3.add(Dense(units=100, activation="relu")) # here we add a dense layer with 100 neurons
ann3.add(Dense(1,activation="sigmoid")) 
ann3.compile(optimizer="adam", loss="binary_crossentropy",metrics=["accuracy"])
ann3.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test))
ann3.evaluate(X_train, y_train)
ann3.evaluate(X_test, y_test)
history=ann3.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test))
ann3.history.history
pd.DataFrame(ann3.history.history).plot(figsize=(15,10))
"""
We can see an expected shape of an overfit model where test accuracy increases to a point and then begins to decrease again.
"""
"""
An overfit model should show accuracy increasing on both train and test and at some point accuracy drops on the test dataset but continues to rise on the training dataset.
"""
"""
Keras provides a weight regularization API that allows you to add a penalty for weight size to the loss function.

Three different regularizer instances are provided; they are:

L1: Sum of the absolute weights.
L2: Sum of the squared weights.
L1L2: Sum of the absolute and the squared weights.
"""
"""
The most common type of regularization is L2, also called simply “weight decay,” with values often on a logarithmic scale between 0 and 0.1, such as 0.1, 0.001, 0.0001, etc.
"""
from keras.regularizers import l2
ann4=Sequential()
ann4.add(Dense(100,activation="relu", kernel_regularizer=l2(0.001)))
ann4.add(Dense(units=100, activation="relu",kernel_regularizer=l2(0.001)))
ann4.add(Dense(units=100, activation="relu",kernel_regularizer=l2(0.001)))
ann4.add(Dense(1,activation="sigmoid")) 
ann4.compile(optimizer="adam", loss="binary_crossentropy",metrics=["accuracy"])
history=ann4.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test))
pd.DataFrame(ann4.history.history).plot(figsize=(15,10))
"""
Thanks to weight decay we got %100 accuracy in the test set after 65 epoch, after that there happens a overfitting.

We will use early stopping to prevent this problem:
"""
from tensorflow.keras.callbacks import EarlyStopping
ann5=Sequential()
ann5.add(Dense(100,activation="relu", kernel_regularizer=l2(0.001)))
ann5.add(Dense(units=100, activation="relu",kernel_regularizer=l2(0.001)))
ann5.add(Dense(units=100, activation="relu",kernel_regularizer=l2(0.001)))
ann5.add(Dense(1,activation="sigmoid")) 
ann5.compile(optimizer="adam", loss="binary_crossentropy",metrics=["accuracy"])
history=ann5.fit(x= X_train, y= y_train, batch_size=32, epochs=100, validation_data=(X_test, y_test),callbacks=[EarlyStopping(monitor="val_accuracy",patience=10)])
pd.DataFrame(ann5.history.history).plot(figsize=(15,10))
# Now we stopped the training after gettin %100 accuracy in the test set
"""
Now we will compare the performance of the deep learning with a normal machine learning model
"""
from sklearn.tree import DecisionTreeClassifier
dtree=DecisionTreeClassifier()
dtree.fit(X_train,y_train)
predictions2=dtree.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix
print(classification_report(y_test,predictions2)) 
"""
As we can obviously see, the deep learning model easily outperform normal machine learning algorithms.
"""