<filename>kmeans_gerint.py<gh_stars>0
# -*- coding: utf-8 -*-
"""kmeans-gerint.ipynb

Automatically generated by Colaboratory.
"""

!pip3 install kmodes
!pip3 install tqdm

# Commented out IPython magic to ensure Python compatibility.
from sklearn.cluster import KMeans
from kmodes.kprototypes import KPrototypes
import pandas as pd
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot as plt
from tqdm import tqdm 
import plotly.graph_objs as go
import seaborn as sns
# %matplotlib inline

from google.colab import drive
drive.mount('/content/drive')

# Defines global variables
categorical_columns = [0,1,2,4,5,6]
total_clusters = 6

# Defines function to filter data
def filter_data(df):
  df = df[df['idade']  <= 120]
  df = df[df['horas_na_fila']  <= 2000]
  df = df[df['horas_na_fila']  >= 0]
  df = df[df['data_solicitacao'] >= '2021-01-01']
  df = df[df['sexo'] != 'NÃO INFORMADO']
  df.dropna(subset = ["idade", "horas_na_fila"], inplace=True)
  return df

# Defines function to normalize data
def normalize_data(df):
  df['sexo'] = df['sexo'].replace(['M'],'MASCULINO')
  df['sexo'] = df['sexo'].replace(['F'],'FEMININO')
  return df

# Defines function to remove unused columns
def remove_unused_columns(df):
  del df['solicitante']
  del df['executante']
  del df['municipio_solicitante']
  del df['municipio_residencia']
  del df['codigo_cid']
  del df['id_usuario']
  del df['data_extracao']
  del df['data_autorizacao']
  del df['data_internacao']
  del df['data_alta']
  del df['data_solicitacao']
  return df

# Defines function to preprocess data
def preprocess_data(df):
  df = filter_data(df)
  df = normalize_data(df)
  df = remove_unused_columns(df)
  return df

# Defines function to load dataset and normalize data
def load_data():
  df = pd.read_csv("/content/drive/MyDrive/Estudos/Unisinos/Disciplinas/Prospecção de Dados e Extração de Conhecimentos/Artigo/gerint_solicitacoes.csv", sep=';')
  df = preprocess_data(df)
  return df

# Defines function plot a chart from a given categoric value
def plot_bar_chart_categorical(column_name, category_name, title, bar_color):
  cluster_labels = []
  cluster_values = []

  for i in range(0, total_clusters):
    df_cluster = df[df['Cluster Labels']==i]
    cluster_labels.append("Cluster " + str(i + 1))
    com_alta = df_cluster[df_cluster[column_name] == category_name]
    cluster_values.append(len(com_alta))
    
  fig = plt.figure(figsize = (10, 5))
  axes = plt.gca()
  axes.set_ylim([0,32000])  
  plt.bar(cluster_labels, cluster_values, color = bar_color, width = 0.4)
  plt.ylabel("Total occurrences")
  plt.title(title + " (" + column_name + " = " + category_name + ")")
  plt.show()

# Prints the head of the dataset
df = load_data()
print(df.shape)
df.head()

#
# Loads sample to generate Elbow Curve.
# The processing happens on a sample of the dataset in order to improve the execution.
df = load_data()
df.shape
df = df[0:5000]
df.shape

#
# Generates the Elbow Curve in order do find out the appropriate number of clusters.
costs = []
n_clusters = []
clusters_assigned = []

for i in tqdm(range(2, 10)):
    try:
        kproto = KPrototypes(n_clusters= i, init='Huang', verbose=2)
        clusters = kproto.fit_predict(df, categorical=categorical_columns)
        costs.append(kproto.cost_)
        n_clusters.append(i)
        clusters_assigned.append(clusters)
    except:
        print(f"Can't cluster with {i} clusters")

#
# Plots the elbow curve from previously gathered data
fig = go.Figure(data=go.Scatter(x=n_clusters, y=costs ))
fig.update_layout(
    autosize=False,
    width=700,
    height=550,
    title={
        'text': "Elbow Method",
        'y':0.9,
        'x':0.5,
        'xanchor': 'center',
        'yanchor': 'top'},
    xaxis_title="Clusters",
    yaxis_title="Costs"
)
fig.show()

# Inspects the categorical variables
df = load_data()
print()
print(df.select_dtypes('object'))
print()
print(df.describe())
print()
print(df.isna().sum())
print()
print("central_regulacao_origem: " + df.central_regulacao_origem.unique())
print("sexo: " + df.sexo.unique())
print("tipo_leito: " + df.tipo_leito.unique())
print("carater: " + df.carater.unique())
print("tipo_internacao: " + df.tipo_internacao.unique())
print(df.shape)
df.head()

# Convert dataframe to matrix
dfMatrix = df.to_numpy()

# Fit the cluster
kprototype = KPrototypes(n_jobs = -1, n_clusters = total_clusters, init = 'Huang', random_state = 0)
kprototype.fit_predict(dfMatrix, categorical = categorical_columns)

# Add the cluster to the dataframe
df['Cluster Labels'] = kprototype.labels_
df['Segment'] = df['Cluster Labels'].map({0:'First', 1:'Second', 2:'Third'})
print("Finalizado k-means clustering")

# Saves to Google Drive
# df.head()
# df.to_csv('dados_com_cluster.csv')
# !cp dados_com_cluster.csv "drive/My Drive/"

# It has already been saved to drive on a previous execution. Simply restores the data!
df = pd.read_csv("/content/drive/MyDrive/dados_com_cluster.csv", sep=',')

#
# Prints the amount of records in each cluser
for i in range(0, total_clusters):
  df_cluster = df[df['Cluster Labels']==i]
  print("Amount of records in cluter " + str(i + 1) + ": " + str(len(df_cluster)))

#
# Loads the amount of records in each cluster
cluster_labels = []
cluster_values = []
for i in range(0, total_clusters):
  df_cluster = df[df['Cluster Labels']==i]
  cluster_labels.append("Cluster " + str(i + 1))
  cluster_values.append(len(df_cluster))

# Plots a chart with total records in each cluster
fig = plt.figure(figsize = (10, 5))
plt.bar(cluster_labels, cluster_values, color ='green', width = 0.4)
plt.ylabel("Total occurrences")
plt.title("Total number of records on each cluster")
plt.show()

# Plots charts for several categorical data
plot_bar_chart_categorical('situacao', 'COM_ALTA', 'Total patients discharged', 'blue')
plot_bar_chart_categorical('situacao', 'INTERNADA', 'Total patients hospitalized', 'blue')

plot_bar_chart_categorical('sexo', 'MASCULINO', 'Total men', 'red')
plot_bar_chart_categorical('sexo', 'FEMININO', 'Total women', 'red')

plot_bar_chart_categorical('central_regulacao_origem', 'EQUIPE DE REGULACAO HOSPITALAR', '', 'orange')
plot_bar_chart_categorical('central_regulacao_origem', 'CENTRAL DE REGULACAO HOSPITALAR DO RIO GRANDE DO SUL', '', 'orange')

plot_bar_chart_categorical('carater', 'URGENCIA', '', 'yellow')
plot_bar_chart_categorical('carater', 'ELETIVA', '', 'yellow')
plot_bar_chart_categorical('carater', 'TRANSFERENCIA', '', 'yellow')

plot_bar_chart_categorical('tipo_internacao', 'PRÓPRIA', '', 'purple')
plot_bar_chart_categorical('tipo_internacao', 'NÃO PRÓPRIA', '', 'purple')

plot_bar_chart_categorical('tipo_leito', 'ENFADULTO', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'UTINEONATAL', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'UTIADULTO', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'ENFPEDIATRICA', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'UTIPEDIATRICA', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'PSIQUIATRICO', '', 'maroon')
plot_bar_chart_categorical('tipo_leito', 'HOSPDIA', '', 'maroon')

#
# Prints information about each categorical data on each cluster
for i in range(1, total_clusters):
  df_cluster = df[df['Cluster Labels']==i]
  print()
  print("--------------")
  print("Info about cluster " + str(i))
  print(df_cluster.shape)

  com_alta = df_cluster[df_cluster['situacao'] == 'COM_ALTA']
  internada = df_cluster[df_cluster['situacao'] == 'INTERNADA']
  print("Total by situacao: [COM ALTA: " + str(len(com_alta)) + "]")
  print("Total by situacao: [INTERNADA: " + str(len(internada)) + "]")

  masculino = df_cluster[df_cluster['sexo'] == 'MASCULINO']
  feminino = df_cluster[df_cluster['sexo'] == 'FEMININO']
  print("Total by sex: [Masculino: " + str(len(masculino)) + "]")
  print("Total by sex: [Feminino: " + str(len(feminino)) + "]")

  equipe_regulacao_hospitalar = df_cluster[df_cluster['central_regulacao_origem'] == 'EQUIPE DE REGULACAO HOSPITALAR']
  central_regulacao_hospitar = df_cluster[df_cluster['central_regulacao_origem'] == 'CENTRAL DE REGULACAO HOSPITALAR DO RIO GRANDE DO SUL']
  sem_central_origem = df_cluster[df_cluster['central_regulacao_origem'] == 'SEM CENTRAL DE ORIGIM']
  print("Total by central_regulacao_origem: [EQUIPE DE REGULACAO HOSPITALAR: " + str(len(equipe_regulacao_hospitalar)) + "]")
  print("Total by central_regulacao_origem: [CENTRAL DE REGULACAO HOSPITALAR DO RIO GRANDE DO SUL: " + str(len(central_regulacao_hospitar)) + "]")
  print("Total by sem_central_origem: [SEM CENTRAL DE ORIGIM: " + str(len(sem_central_origem)) + "]")

  urgencia = df_cluster[df_cluster['carater'] == 'URGENCIA']
  eletiva = df_cluster[df_cluster['carater'] == 'ELETIVA']
  transferencia = df_cluster[df_cluster['carater'] == 'TRANSFERENCIA']
  print("Total by carater: [Urgencia: " + str(len(urgencia)) + "]")
  print("Total by carater: [Eletiva: " + str(len(eletiva)) + "]")
  print("Total by carater: [Transferencia: " + str(len(transferencia)) + "]")

  obstetrico = df_cluster[df_cluster['tipo_leito'] == 'OBSTETRICO']
  enfadulto = df_cluster[df_cluster['tipo_leito'] == 'ENFADULTO']
  utineonatal = df_cluster[df_cluster['tipo_leito'] == 'UTINEONATAL']
  utiadulto = df_cluster[df_cluster['tipo_leito'] == 'UTIADULTO']
  enfpediatrica = df_cluster[df_cluster['tipo_leito'] == 'ENFPEDIATRICA']
  utipediatrica = df_cluster[df_cluster['tipo_leito'] == 'UTIPEDIATRICA']
  psiquiatrico = df_cluster[df_cluster['tipo_leito'] == 'PSIQUIATRICO']
  hospdia = df_cluster[df_cluster['tipo_leito'] == 'HOSPDIA']
  print("Total by tipo_leito: [OBSTETRICO: " + str(len(obstetrico)) + "]")
  print("Total by tipo_leito: [ENFADULTO: " + str(len(enfadulto)) + "]")
  print("Total by tipo_leito: [UTINEONATAL: " + str(len(utineonatal)) + "]")
  print("Total by tipo_leito: [UTIADULTO: " + str(len(utiadulto)) + "]")
  print("Total by tipo_leito: [ENFPEDIATRICA: " + str(len(enfpediatrica)) + "]")
  print("Total by tipo_leito: [UTIPEDIATRICA: " + str(len(utipediatrica)) + "]")
  print("Total by tipo_leito: [PSIQUIATRICO: " + str(len(psiquiatrico)) + "]")
  print("Total by tipo_leito: [HOSPDIA: " + str(len(hospdia)) + "]")


  propria = df_cluster[df_cluster['tipo_internacao'] == 'PRÓPRIA']
  nao_propria = df_cluster[df_cluster['tipo_internacao'] == 'NÃO PRÓPRIA']
  print("Total by tipo_internacao: [PROPRIA: " + str(len(propria)) + "]")
  print("Total by tipo_internacao: [NAO_PROPRIA: " + str(len(nao_propria)) + "]")


  print("Total por faixa de idade")
  out = df_cluster.apply(lambda x: pd.cut(df_cluster.idade, bins=[0,10,20,30,40,50,60,70,80,90,100,110,120,130]).value_counts()).add_prefix('count_')
  print(out.count_idade)

#
# Prints the number of people, grouped by 10 years in each cluster
for i in range(0, total_clusters):
  df_cluster = df[df['Cluster Labels']==i]
  out = df_cluster.apply(lambda x: pd.cut(df_cluster.idade, bins=[0,10,20,30,40,50,60,70,80,90,100,110,120,130]).value_counts()).add_prefix('count_')
  print(out.count_idade)

#
# Plots the number of people, grouped by 10 years in each cluster
cluster_labels = []
cluster_values = []
for i in range(0, total_clusters):
  df_cluster = df[df['Cluster Labels']==i]
  # Plot por idade, de 10 em 10 anos
  out = df_cluster.apply(lambda x: pd.cut(df_cluster.idade, bins=[0,10,20,30,40,50,60,70,80,90,100,110,120,130]).value_counts()).add_prefix('count_')

#
# Manually assembles the chart based on the data extracted above.
# (some clusters do not have values ​​in all age ranges, which would result in Python error in case of automatic append)
X = ['(0, 9)','(10, 19)','(20, 29)','(30, 39)','(40, 49)','(50, 59)','(60, 69)','(70, 79)','(80, 89)','(90, 99)','(100, 109)','(110, 119)']
cluster_values.append([0, 0, 0, 0, 3580, 7510, 8912, 6168, 2857, 428, 12, 1])
cluster_values.append([119, 126, 193, 235, 251, 317, 318, 185, 59, 6, 0, 0])
cluster_values.append([27, 12, 10, 31, 43, 59, 64, 29, 9, 0, 0, 0])
cluster_values.append([2087, 3112, 6944, 6520, 2404, 0, 0, 0, 0, 0, 0, 0])
cluster_values.append([49, 37, 71, 75, 100, 108, 152, 66, 19, 0, 0, 0])
cluster_values.append([92, 210, 386, 626, 679, 922, 941, 642, 267, 45, 1, 0])

# Plots the chart itself
X_axis = np.arange(len(X))

fig = plt.figure(figsize = (7, 3))
plt.bar(X_axis - 0.4, cluster_values[0], 0.1, label = 'Cluster 1')
plt.bar(X_axis - 0.3, cluster_values[1], 0.1, label = 'Cluster 2')
plt.bar(X_axis - 0.2, cluster_values[2], 0.1, label = 'Cluster 3')
plt.bar(X_axis + 0.2, cluster_values[3], 0.1, label = 'Cluster 4')
plt.bar(X_axis + 0.3, cluster_values[4], 0.1, label = 'Cluster 5')
plt.bar(X_axis + 0.4, cluster_values[5], 0.1, label = 'Cluster 6')
  
plt.xticks(X_axis, X, rotation='vertical')
plt.xlabel("Age (grouped into 10 years)")
plt.ylabel("Number of people")
plt.title("Number of people in each cluster")
plt.legend()
plt.show()