# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python [conda env:dm]
#     language: python
#     name: conda-env-dm-py
# ---

# +
import math
import pandas as pd

from sklearn import metrics, preprocessing, cluster

from scipy.spatial import distance
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.cm as cm

# %matplotlib inline

# +
# Read file
icfes = pd.read_csv("./data/icfes_preprocessing.csv", low_memory=False, encoding="utf-8")

print(icfes.shape)

icfes.describe()

# +
# Select nominal columns

columns = ['COLE_AREA_UBICACION', 'COLE_BILINGUE', 'FAMI_COMECARNEPESCADOHUEVO', 
    'FAMI_COMECEREALFRUTOSLEGUMBRE', 'FAMI_COMELECHEDERIVADOS', 
    'FAMI_CUARTOSHOGAR', 'FAMI_EDUCACIONMADRE', 
    'FAMI_EDUCACIONPADRE', 'FAMI_ESTRATOVIVIENDA', 
    'FAMI_NUMLIBROS', 'FAMI_PERSONASHOGAR', 
    'FAMI_SITUACIONECONOMICA', 'FAMI_TIENEAUTOMOVIL', 
    'FAMI_TIENECOMPUTADOR', 'FAMI_TIENECONSOLAVIDEOJUEGOS', 
    'FAMI_TIENEHORNOMICROOGAS', 'FAMI_TIENELAVADORA', 
    'FAMI_TIENEMOTOCICLETA', 'FAMI_TIENESERVICIOTV', 
    'FAMI_TRABAJOLABORMADRE', 'FAMI_TRABAJOLABORPADRE',
    'DESEMP_INGLES'
]

dataset = icfes[columns]

# Filter by value
subset = dataset.loc[dataset['DESEMP_INGLES'].isin(["B+"])]

print("dataset", dataset.shape, "subset", subset.shape)

# Nominal to binary
dummies = pd.get_dummies(subset, sparse=True)

#dummies.to_csv("./data/icfes_pre_clustering.csv", index=False, encoding="utf-8")

rows = math.ceil(dummies.shape[0]*1.0)

X = dummies.values[:rows, :]

print("X", X.shape)
# -

#
# # K-modes

# +
from kmodes.kmodes import KModes
from kmodes.kprototypes import KPrototypes

class Clustering:
    
    def __init__(self, clusterer, cluster_labels):
        self.clusterer = clusterer
        self.cluster_labels = cluster_labels

def find_kmodes(X, range_n_clusters):
    kmodes = []
    
    for n_clusters in range_n_clusters:
        # Initialize the clusterer with n_clusters value.
        clusterer =  KModes(n_clusters=n_clusters, init='Cao', n_init=5)
        cluster_labels = clusterer.fit_predict(X)
        
        kmodes.append(Clustering(clusterer, cluster_labels))
        
        print("Kmodes({0}) done.".format(n_clusters))
    
    return kmodes

range_n_clusters = range(2, 6)

kmodes = find_kmodes(X, range_n_clusters)


# +
def plot_silhouette_score(X, clusterings, range_n_clusters):
        
    silhouette_avg = []
    
    for clustering in clusterings:
        # The silhouette_score gives the average value for all the samples.
        # This gives a perspective into the density and separation of the formed clusters
        current_silhouette_avg = metrics.silhouette_score(X, clustering.cluster_labels, metric='hamming', n_jobs=-1)
        
        silhouette_avg.append(current_silhouette_avg)
        n_clusters= len(np.unique(clustering.cluster_labels))
        
        print("For n_clusters =", n_clusters, 
              "The average silhouette_score is :", current_silhouette_avg)
    
    fig, ax1 = plt.subplots(1, 1, figsize=(16, 9))
    ax1.plot(range_n_clusters, silhouette_avg, label='kmodes')
    ax1.legend()
    ax1.set_title("K-modes silhouette score on ICFES dataset")
    ax1.set_xlabel("Cluster size")
    ax1.set_ylabel("Silhouette score")
    
    plt.show()
    
plot_silhouette_score(X, kmodes, range_n_clusters)


# +
def plot_silhouette(X, cluster_labels, metric='hamming'):
    
    n_clusters = len(np.unique(cluster_labels))

    # The silhouette_score gives the average value for all the samples.
    # This gives a perspective into the density and separation of the formed clusters
    silhouette_avg = metrics.silhouette_score(X, cluster_labels, metric=metric, n_jobs=-1)
    print("For n_clusters =", n_clusters, "The average silhouette_score is :", silhouette_avg)

    # Compute the silhouette scores for each sample
    sample_silhouette_values = metrics.silhouette_samples(X, cluster_labels, metric=metric, n_jobs=-1)
    
    # Create a subplot with 1 row and 1 columns
    fig, (ax1) = plt.subplots(1, 1)
    fig.set_size_inches(16, 9)

    # The silhouette coefficient can range from -1, 1 but in this example all lie within [-0.1, 1]
    ax1.set_xlim([-0.1, 1])
    # The (n_clusters+1)*10 is for inserting blank space between silhouette plots of individual clusters.
    ax1.set_ylim([0, len(X) + (n_clusters + 1) * 10])

    y_lower = 10
    for i in range(n_clusters):
        # Aggregate the silhouette scores for samples belonging to
        # cluster i, and sort them
        ith_cluster_silhouette_values = sample_silhouette_values[cluster_labels == i]

        ith_cluster_silhouette_values.sort()

        size_cluster_i = ith_cluster_silhouette_values.shape[0]
        y_upper = y_lower + size_cluster_i

        color = plt.cm.Set1(float(i) / n_clusters)
        ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values,
                              facecolor=color, edgecolor=color, alpha=0.7)

        # Label the silhouette plots with their cluster numbers at the middle
        ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i))

        # Compute the new y_lower for next plot
        y_lower = y_upper + 10  # 10 for the 0 samples

    ax1.set_title("The silhouette plot for the various clusters.")
    ax1.set_xlabel("The silhouette coefficient values")
    ax1.set_ylabel("Cluster label")

    # The vertical line for average silhouette score of all the values
    ax1.axvline(x=silhouette_avg, color="red", linestyle="--")

    ax1.set_yticks([])  # Clear the yaxis labels / ticks
    ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1])

    plt.suptitle(("Silhouette analysis for KModes clustering "
                      "with {0} clusters".format(n_clusters)),fontsize=14, fontweight='bold')
    
    plt.show()
    
clustering = kmodes[0]

plot_silhouette(X, clustering.cluster_labels)


# +
def label_centroids(clusterer, attribute_names):    
    label_atributes = np.array(attribute_names)
    
    for centroid in clusterer.cluster_centroids_:
        # Label the centroids
        label_idx = np.array(centroid, dtype=bool)
        label = label_atributes[label_idx]
        print(label)
        
label_centroids(clustering.clusterer, dummies.columns)
# -

# # Hierarchical

# +
from scipy.cluster import hierarchy    

labels = dummies.columns

methods = {'single', 'complete', 'average'}

def hierarchical_exp(X, methods):
    
    Zs = []
    
    for method in methods:
        # generate the linkage matrix
        Zs.append(hierarchy.linkage(X, method=method, metric='hamming'))
        print("Done for:", method)

    return Zs
    
Zs = hierarchical_exp(X, methods)
# -

for k in np.arange(2, 6):
    cluster_labels = hierarchy.fcluster(Zs[0], k, criterion='maxclust')
    
    labels = np.unique(cluster_labels)
        
    if labels.size > 1:
    
        silhouette_score = metrics.silhouette_score(X, cluster_labels, metric='hamming', n_jobs=-1)
    
        print(k, silhouette_score)


# +
def plot_hierarchical(Z, methods):
    r, c = (2, 2)
    fig, axes = plt.subplots(r, c, figsize=(16, 9*r))
    plot_num = 1

    for z, method in zip(Z, methods):
        # plot
        ax = plt.subplot(r, c, plot_num)
        dn = hierarchy.dendrogram(z, p=7, truncate_mode='lastp')
        ax.set_title("Hierarchical clustering ("+method+")")
        ax.set_ylabel('Distance')
        plot_num +=1

    plt.tight_layout()
    plt.show()
    
plot_hierarchical(Zs, methods)


# -

# # DBSCAN

# +
def dbscan_experiment(X):

    epsilons = np.linspace(0.01, 0.5, num=20)
    n_clusters = []
    clusterers = []
    
    for eps in epsilons:
        db = cluster.DBSCAN(eps=eps, min_samples=4, metric='hamming', n_jobs=-1).fit(X)
        labels = db.labels_
        clusterers.append(db)

        # Number of clusters in labels, ignoring noise if present.
        n_clusters.append(len(set(labels)) - (1 if -1 in labels else 0))
    
    return clusterers, n_clusters

clusterers, n_clusters = dbscan_experiment(X)


# +
def plot_dbscan_experiment(clusterers, n_clusters):
    
    epsilons = [db.eps for db in clusterers]
    
    # Plot result
    fig, ax = plt.subplots(1, 1, figsize=(16, 9))

    ax.plot(epsilons, n_clusters)
    
    ax.set_xticks(epsilons)
    
    # Annotate coordinates
    plt.plot(epsilons, n_clusters)
    for xy in zip(epsilons, n_clusters):
        ax.annotate("({0})".format(xy[1]), xy=xy, textcoords='data')
    
    ax.set_title("DBSCAN on ICFES dataset")
    ax.set_xlabel("Epsilon")
    ax.set_ylabel("Estimated number of clusters")
    
    plt.show()
    
#############

plot_dbscan_experiment(clusterers, n_clusters)
# -



StarcoderdataJupyterScriptsDedupFiltered