# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="xpzAIXE0-1MJ"
# # Decision Trees
# <NAME> A01038061

# + [markdown] id="7C3E4zYxDkE9"
# ## Configs

# + id="H6e0AXZA-0vX"
#Python >= 3.5 required
import sys
assert sys.version_info >= (3,5)
#Scikit-Learn >= 0.20 is required
import sklearn
assert sklearn.__version__ >="0.20"
#imports
import numpy as np
import pandas as pd
import matplotlib as mpl
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from graphviz import Source
from sklearn.tree import export_graphviz
from sklearn.tree import DecisionTreeClassifier
from matplotlib.colors import ListedColormap
from sklearn.linear_model import LogisticRegression
import os

# + id="VZRugjAT_cRz"
#Configs dirs for images
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)
PROJECT_ROOT_DIR = "."
CHAPTER_ID = "decision_trees"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID,)
os.makedirs(IMAGES_PATH, exist_ok=True)


# + id="QpXLLrUHAvId"
#save fig functions on the directory
def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
  path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
  print("Saving figure", fig_id)
  if tight_layout:
    plt.tight_layout()
  plt.savefig(path, format=fig_extension, dpi=resolution)


# + id="3jSmz6h1u7Zg"
#Batch Gradient Descent Algorithm
def sigmoid(x):
  return 1.0 / (1 + np.exp(-x))

def lr_hypothesis(x, theta):
  return np.dot(x,theta)

#return hb_opt => 
def batchGradientDescent(X, y, b0 = 0.5, ALPHA = 0.25, max_it=5000, 
                         threshold = 1 * pow(10,-4)):
  
  #prepare data
  X = X.values
  y = y.values
  
  zm, zn = X.shape
  z = np.ones(zm)
  z = z.reshape(zm, 1)
  X = np.append(z,X,axis=1)

  m, n = X.shape
  theta = np.zeros(n) * b0
  theta = theta.reshape(n,1)
  y = y.reshape(-1,1)
  diff = 1
  j = 0
  
  while j < max_it and diff > threshold:
    last_t = theta
    infunc1 = sigmoid(lr_hypothesis(X, theta)) - y
    gradient = np.dot(X.T, infunc1) / m
    theta = theta - (ALPHA / m) * gradient
    diff = np.linalg.norm(last_t-theta)
    j+=1
  return theta, j


# + id="AwTBdy2zvA9P"
#Testing functions

#return if classify in 1 or 0.
def classify(x):
  return int(x > 0.5)

#compare data
def compare(y_hat, y):
  return np.where(y_hat == y, 0, 1)

#return error
def error(y_hat, y, T):
  return 1 / T * sum(compare(y_hat, y))

#Apply model with values to predict probability of 1.
def predict(model, X):
  X = X.values
  X = np.insert(X, 0, 1.0)
  return sigmoid(np.dot(model.T, X))


# + [markdown] id="MAH53HTQD97d"
# ## Iris dataset
#

# + id="JkW9PiY_Byev"
#import dataset and transform to csv
from sklearn.datasets import load_iris
iris = load_iris()
X = iris.data[:, 2:] # work only with petal length and width
y = iris.target

# + id="fiZRiKPx-9ui"
#Split data for traning and testing
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

# + colab={"base_uri": "https://localhost:8080/", "height": 441} id="mhwIPoUeFTow" outputId="e8c52840-e108-4c5f-a45a-17b208fe62a4"
#Decision tree fit with the training data
tree_clf = DecisionTreeClassifier(max_depth = 2, random_state=0)
tree_clf = tree_clf.fit(X_train, y_train)

#Create tree view
export_graphviz(
    tree_clf,
    out_file = os.path.join(IMAGES_PATH, "iris_tree.dot"),
    feature_names = iris.feature_names[2:],
    class_names = iris.target_names,
    rounded=True,
    filled=True
)

Source.from_file(os.path.join(IMAGES_PATH, "iris_tree.dot"))

# + colab={"base_uri": "https://localhost:8080/"} id="EsXIMmp7A6KQ" outputId="8eab3446-103b-46fb-c6b4-2d04472d1879"
#Calculate Accuracy: correct predictions / total number of data points
score = tree_clf.score(X_test, y_test)
print("Score of iris dataset with Decision tree model: ", score)


# + colab={"base_uri": "https://localhost:8080/", "height": 314} id="eRzn3cwai54t" outputId="d4418d03-e796-4256-f3d2-9c97230fc0ba"
#Visualization by each split: need to work only with binary classifications
def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
  x1s = np.linspace(axes[0], axes[1], 100)
  x2s = np.linspace(axes[2], axes[3], 100)
  x1, x2 = np.meshgrid(x1s, x2s)
  X_new = np.c_[x1.ravel(), x2.ravel()]
  y_pred = clf.predict(X_new).reshape(x1.shape)
  custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
  plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
  if not iris:
    custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
    plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
  if plot_training:
    plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris setosa")
    plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris versicolor")
    plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris virginica")
    plt.axis(axes)
  if iris:
    plt.xlabel("Petal length", fontsize=14)
    plt.ylabel("Petal width", fontsize=14)
  else:
    plt.xlabel(r"$x_1$", fontsize=18)
    plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
  if legend:
    plt.legend(loc="lower right", fontsize=14)

plt.figure(figsize=(8, 4))
plot_decision_boundary(tree_clf, X_test, y_test)
plt.plot([2.45, 2.45], [0, 3], "k-", linewidth=2)
plt.plot([2.45, 7.5], [1.75, 1.75], "k--", linewidth=2)
plt.text(1.40, 1.0, "Depth=0", fontsize=15)
plt.text(3.2, 1.80, "Depth=1", fontsize=13)
save_fig("decision_tree_decision_boundaries_plot")
plt.show()

# + colab={"base_uri": "https://localhost:8080/"} id="uyckqE0GyUf7" outputId="d032893e-bc81-4851-9e04-e9b4e0064d21"
print("Iris Figs generated in: ", IMAGES_PATH)

# + [markdown] id="JeKhNHycFVV8"
# ## Wine dataset
#

# + id="vsFriFRiGnhd"
#import dataset and transform to csv
from sklearn.datasets import load_wine
wine = load_wine()
df_wine = pd.DataFrame(data=wine['data'], columns=wine['feature_names'])
df_wine['target'] = wine['target']

# + colab={"base_uri": "https://localhost:8080/", "height": 215} id="LucOkjhooNm6" outputId="88c2516e-0a93-490f-da92-5cfb8e596ce5"
df_wine.sample(5)

# + colab={"base_uri": "https://localhost:8080/", "height": 304} id="ERIeoNtboTfj" outputId="e5d0b10d-869c-4d11-ce82-29bfcbc839c7"
df_wine.describe()

# + id="6l0cL9O-oZOA"
#Split data for traning and testing
featuresNames = df_wine.columns[:-1]
classNames = 'target'
X_train, X_test, y_train, y_test = train_test_split(df_wine[featuresNames], df_wine[classNames], test_size=0.33, random_state=42)

# + colab={"base_uri": "https://localhost:8080/"} id="Rbgs1VY55MuN" outputId="708cda65-b4f0-417d-fe8c-957f534940e5"
wine.target_names

# + colab={"base_uri": "https://localhost:8080/", "height": 441} id="zlejJL65pQFl" outputId="afae0061-610c-4dc1-aad5-31c2e86adeba"
#Decision tree fit with the training data
tree_clf = DecisionTreeClassifier(max_depth = 2, random_state=0)
tree_clf = tree_clf.fit(X_train, y_train)

#Create tree view
export_graphviz(
    tree_clf,
    out_file = os.path.join(IMAGES_PATH, "wine_tree.dot"),
    feature_names = featuresNames,
    class_names = wine.target_names,
    rounded=True,
    filled=True
)

Source.from_file(os.path.join(IMAGES_PATH, "wine_tree.dot"))

# + colab={"base_uri": "https://localhost:8080/"} id="AvL1wpffqDnr" outputId="5bcafeac-84a8-4632-bcdb-7ed24592c5ac"
#Calculate Accuracy: correct predictions / total number of data points
score = tree_clf.score(X_test, y_test)
print("Score of wine dataset Decision Tree model: ", score)

# + colab={"base_uri": "https://localhost:8080/"} id="i5IRVgyiyw8T" outputId="a5687b2c-26d0-4dda-823c-b98ee0844a3c"
print("Wine Figs generated in: ", IMAGES_PATH)

# + [markdown] id="218AecS7qjlz"
# ## Breast Cancer dataset
#

# + id="fODlZLNsqpAI"
from sklearn.datasets import load_breast_cancer
cancer = load_breast_cancer()
df_cancer = pd.DataFrame(data=cancer['data'], columns=cancer['feature_names'])
df_cancer['target'] = cancer['target']

# + colab={"base_uri": "https://localhost:8080/", "height": 249} id="lsZRUBNOrNjq" outputId="9a4068e5-31d8-4078-d89d-4eecdb8586d1"
df_cancer.sample(5)

# + colab={"base_uri": "https://localhost:8080/", "height": 338} id="byo-KolirTcj" outputId="6a297808-f094-411d-d774-3f648752b180"
df_cancer.describe()

# + id="dDj0j7cTrVtx"
#Split data for traning and testing
featuresNames = df_cancer.columns[:-1]
classNames = 'target'
X_train, X_test, y_train, y_test = train_test_split(df_cancer[featuresNames], df_cancer[classNames], test_size=0.33, random_state=42)

# + colab={"base_uri": "https://localhost:8080/", "height": 441} id="6uTGuAYirezq" outputId="8d9e7142-e75f-4982-eb25-0c85e194200f"
#Decision tree fit with the training data
tree_clf = DecisionTreeClassifier(max_depth = 2, random_state=0)
tree_clf = tree_clf.fit(X_train, y_train)

#Create tree view
export_graphviz(
    tree_clf,
    out_file = os.path.join(IMAGES_PATH, "cancer_tree.dot"),
    feature_names = featuresNames,
    class_names = cancer.target_names,
    rounded=True,
    filled=True
)

Source.from_file(os.path.join(IMAGES_PATH, "cancer_tree.dot"))

# + colab={"base_uri": "https://localhost:8080/"} id="ZkepYfLGryVy" outputId="a3b14ee9-43c8-4743-a857-f3979857894b"
#Calculate Accuracy: correct predictions / total number of data points
score = tree_clf.score(X_test, y_test)
print("Score of cancer dataset with Decision tree model: ", score)

# + colab={"base_uri": "https://localhost:8080/"} id="wcFhLkZYy4Vv" outputId="dd65f1f4-cc05-4b64-9d9f-803450b41b16"
print("Cancer Figs generated in: ", IMAGES_PATH)

# + colab={"base_uri": "https://localhost:8080/"} id="S_VqLEXPx0TY" outputId="01fe33ef-a28a-4ef6-936f-fa9f36325403"
#Use logisticRegression
model, j = batchGradientDescent(X_train, y_train, 0.5, 0.05, 1000, 0.0001)
print('Logistic Regression Batch Gradient model B vector: ')
print(model)
print('Logistic Regression Batch Gradient model iterations: ', j)

# + colab={"base_uri": "https://localhost:8080/"} id="jBdNXaz2x2Ga" outputId="8f4a7fa3-f876-415f-f16b-222e1b13f2ad"
#Test model

#predict values
y_predicted_value = X_test.apply(lambda x: predict(model, x), axis=1)
#Classify values
y_predicted = y_predicted_value.apply(classify)
#error
print('error of cancer dataset with BatchGradient model: ', error(y_predicted, y_test, y_predicted.size))
print('score  of cancer dataset with BatchGradient model: ', 1-error(y_predicted, y_test, y_predicted.size))

# + colab={"base_uri": "https://localhost:8080/"} id="0onDfrp2BZ_1" outputId="ebf6bf62-2515-4650-ef83-ec8242b600f7"
#Logistic Regression with SciKit-Learn
model = LogisticRegression(max_iter=10000)
model.fit(X_train, y_train)
print("coef of cancer dataset with SciKit-Learn model:", model.coef_)
print("bias od cancer dataset with SciKit-Learn model:", model.intercept_)
print("max_iters of model:", 10000)

# + colab={"base_uri": "https://localhost:8080/"} id="myxiNfqoBeII" outputId="2f9e5271-d705-4920-8311-b6c5135f4c3b"
#error with SciKit-Learn
y_hat = model.predict(X_test)
print('error of cancer dataset with SciKit-Learn model:', error(y_hat, y_test, y_hat.size))
print('Score of cancer dataset SciKit-Learn model:', 1-error(y_hat, y_test, y_hat.size))

StarcoderdataJupyterScriptsDedupFiltered