text: # -*- coding: utf-8 -*-
  """DataUndersting&Clustering.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1e82x0eFoEwTJA1oEgwCshhCZ766c-4dJ
  
  #  1 Data Understading & Preparation
  """
  
  import numpy as np
  import pandas as pd
  import matplotlib.pyplot as plt
  import seaborn as sb
  
  from matplotlib.ticker import AutoMinorLocator
  from matplotlib import gridspec
  
  #scaling, normalization
  from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler
  #kmeans, dbscan, hierarchical (sklearn)
  from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
  #evaluation
  from sklearn.metrics import silhouette_score
  from sklearn import metrics
  
  #distance matrix (dbscan elbow, hierarchical)
  from scipy.spatial.distance import pdist, squareform
  # hierarchical (scipy)
  from scipy.cluster.hierarchy import linkage, dendrogram
  #seaborn #bokeh #altair
  from sklearn.decomposition import PCA
  
  from yellowbrick.cluster import KElbowVisualizer
  from google.colab import files
  
  !pip install pyfim
  from fim import apriori
  
  import plotly.express as px
  
  #caricamento del dataset
  df = pd.read_csv('words_glasgow.csv')
  #faccio una copia del dataset in caso di manipolazione dati
  dfcopy= df.copy()
  
  #visualizzazione di alcune righe per avere un'idea dei dati
  #le prime righe
  df.head()
  
  #le ultime righe
  df.tail()
  
  #visualizzazione della dimensione 
  df.shape
  
  #prima ricognizione indicativa sui dati
  df.describe()
  
  df.describe(include=['O'])
  
  df.info()
  
  """In this dataset there are 4682 unique words. Each word is examined in regards of different variables. Those variables are lenght, arousal, valence, dominance, concretness, imageability, familiarity, age of aquisition, semsize gender, polysemy and frequence in a google web corpus.`
  
  ## 1.1 Data Semantics
  """
  
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  
  first=df.loc[:,"familiarity":]
  aoa_descr=first.describe()
  
  print(aoa_descr.to_latex())
  
  """testo in grassetto
  
  #### Arousal
  
  Brief description of the variables
  """
  
  #NOME VARIABILE DI INTERESSE
  refvar="arousal"
  
  
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,10),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """Other comments
  
  #### Valence
  
  Brief description of the variables
  """
  
  #NOME VARIABILE DI INTERESSE
  refvar="valence"
  
  
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,10),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  
  
  """Other comments"""
  
  
  
  """#### Dominance
  
  Brief description of the variables
  """
  
  refvar="dominance"
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,10),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """Other comments
  
  #### Concreteness
  
  (numerical): represents the degree to which something can be experienced by our senses. It ranges from 1, abstract words, to 7, concrete words.
  """
  
  #NOME VARIABILE DI INTERESSE
  refvar="concreteness"
  
  
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """Other comments"""
  
  
  
  """#### Imageability
  
  (numerical): measures how difficult is to generate a mental image of something. It ranges from 1, hard to imagine, to 7, easy to imagine.
  """
  
  #NOME VARIABILE DI INTERESSE
  refvar="imageability"
  
  
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """Other comments"""
  
  
  
  """#### Familiarity
  
  (numerical): is a measure of a word’s subjective experience. The range is from 1 (unfamiliar) to 7 (familiar).
  """
  
  #NOME VARIABILE DI INTERESSE
  refvar="familiarity"
  
  
  
  #calcolo frequenza assoluta per aoa
  df[refvar].value_counts()
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  boxplot_str="boxplot"
  str1="distribution of"+" "+refvar
  str2=boxplot_str+" "+str1
  plt.title(str2, size = 18)
  aoa_boxplot.set_xlabel(refvar, rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title(str1, size = 18)
  plt.xlabel(xlabel = refvar, fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """Other comments"""
  
  
  
  """#### Age of Aquisition
  
  The variable age of aquisition (aoa) indicates the supposed age in which that person first learned that specific word. Clearly it is not easy to remember at which age a word is learned, and for that reason in this dataset this variables refears to an estimation of the age of aquisition. The scale is defined as a series of consecutive 2-year periods from the ages of 2 and 12 years, and a final period referring to 13 years and older. This shows how we have 7 different ranges, 0-2, 2-4, 4-6, 6-8, 8-10-, 10-12 and 13+.
  """
  
  #calcolo frequenza assoluta per aoa
  df["aoa"].value_counts()
  
  #imposto lo spazio
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  plt.rcParams["figure.figsize"] = [16, 4]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  plt.subplot(1, 2, 1)
  #distribuzione e tendenze centrali: boxplot
  aoa_boxplot = sb.boxplot( x = "aoa", data=df, color = colors1[1])
  plt.title("Boxplot distribution of age of aquisition", size = 18)
  aoa_boxplot.set_xlabel("Age of aquisition", rotation = "horizontal", size = 16)
  plt.subplot(1, 2, 2)
  
  aoa1= df["aoa"].dropna(0, inplace = False)
  
  #imposto la legenda (da rivedere)
  #lables_aoa= [0-2, 2-4, 4-6, 6-8, 8-10, 10-12, 13]
  #plt.legend(title="Age of aquisition ranges", title_fontsize = 14, bbox_to_anchor=(1, 1), labels=lables_aoa )
  
  #distribuzione e tendenze centrali: istogramma
  aoa_hist = plt.hist(aoa1,  color = colors1[1],edgecolor='black',bins=range(0,8))
  
  plt.title("Distribution of aoa", size = 18)
  plt.xlabel(xlabel = "Age of Aquisition", fontsize = 16)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  
  plt.show()
  
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = "aoa", data=df, color = colors1[1])
  plt.title("Boxplot distribution of age of aquisition", size = 18)
  aoa_boxplot.set_xlabel("Age of aquisition", rotation = "horizontal", size = 16)
  
  plt.show()
  
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = "aoa", data=df, color = colors1[1])
  plt.title("Boxplot distribution of Age of Aquisition", size = 18)
  bin_centers = 0.5 * np.diff(bins) + bins[:-1]
  for i, x in zip(range(0,7), bin_centers):
      # Label the raw counts
      age_range=['0-2','2-4','4-6','6-8','8-10','10-12','13+']
      ax.annotate(age_range[i], xy=(x, 0), xycoords=('data', 'axes fraction'),
          xytext=(0, -22), textcoords='offset points',fontsize=16, va='top', ha='center')
  
  
  aoa_boxplot.set_xlabel("Age of Aquisition", rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df["aoa"].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(0,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  bin_centers = 0.5 * np.diff(bins) + bins[:-1]
  for i, x in zip(range(0,7), bin_centers):
      # Label the raw counts
      age_range=['0-2','2-4','4-6','6-8','8-10','10-12','13+']
      ax.annotate(age_range[i], xy=(x, 0), xycoords=('data', 'axes fraction'),
          xytext=(0, -22), textcoords='offset points',fontsize=16, va='top', ha='center')
  
      # Label the percentages
      #percent = '%0.0f%%' % (100 * float(count) / counts.sum())
      #ax.annotate(percent, xy=(x, 0), xycoords=('data', 'axes fraction'),
          #xytext=(0, -32), textcoords='offset points', va='top', ha='center')
  
  
  # Give ourselves some more room at the bottom of the plot
  #plt.subplots_adjust(bottom=0.50)
  plt.title("Distribution of aoa", size = 18)
  plt.xlabel(xlabel = "Age of Aquisition", fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df["aoa"].describe()
  
  print(aoa_descr.to_latex())
  
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  
  refvar="aoa"
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  
  plt.title("Boxplot distribution of aoa", size = 18)
  aoa_boxplot.set_xlabel("Age of Aquisition", rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  bin_centers = 0.5 * np.diff(bins) + bins[:-1]
  for i, x in zip(range(0,7), bin_centers):
      # Label the raw counts
      age_range=['0-2','2-4','4-6','6-8','8-10','10-12','13+']
      ax.annotate(age_range[i], xy=(x, 0), xycoords=('data', 'axes fraction'),
          xytext=(0, -22), textcoords='offset points',fontsize=16, va='top', ha='center')
  
      # Label the percentages
      #percent = '%0.0f%%' % (100 * float(count) / counts.sum())
      #ax.annotate(percent, xy=(x, 0), xycoords=('data', 'axes fraction'),
          #xytext=(0, -32), textcoords='offset points', va='top', ha='center')
  
  
  # Give ourselves some more room at the bottom of the plot
  plt.subplots_adjust(bottom=0.50)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title("Distribution of aoa", size = 18)
  plt.xlabel(xlabel = "Age of Aquisition", fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  
  
  aoa_descr=df["aoa"].describe()
  
  print(aoa_descr.to_latex())
  
  """There are 4682 occurrences for this variable. That means that there are no missing values, or NaN. The mean is 4.14, and the standard deviation is 1.25.
  
  #### Size
  
  In this dataset "size" is a measure of magnitude expressed in either concrete or abstarct terms (big, small). That is, if a word can be associated with adjectives like big or small (e.g. palace or mountain for concrete object, and knowledge or love for abstarct ideas).
  """
  
  #calcolo frequenza assoluta per aoa
  df["semsize"].value_counts()
  
  #imposto lo spazio
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  plt.rcParams["figure.figsize"] = [16, 4]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  plt.subplot(1, 2, 1)
  #distribuzione e tendenze centrali: boxplot
  aoa_boxplot = sb.boxplot( x = "semsize", data=df, color = colors1[1])
  plt.title("Boxplot distribution of semantic size", size = 18)
  aoa_boxplot.set_xlabel("semantic size", rotation = "horizontal", size = 16)
  plt.subplot(1, 2, 2)
  
  size1= df["semsize"].dropna(0, inplace = False)
  
  #imposto la legenda (da rivedere)
  #lables_size= [0-2, 2-4, 4-6, 6-8, 8-10, 10-12, 13]
  #plt.legend(title="Age of aquisition ranges", title_fontsize = 14, bbox_to_anchor=(1, 1), labels=lables_size )
  #distribuzione e tendenze centrali: istogramma
  aoa_hist = plt.hist(size1,  color = colors1[1],edgecolor='black',bins=range(1,8))
  
  #BELLURIE
  
  plt.title("Distribution of semantic size", size = 18)
  plt.xlabel(xlabel = "semantic size", fontsize = 16)
  plt.ylabel(ylabel = "Degree", fontsize = 16)
  sb.despine(right = True)
  
  plt.show()
  
  
  
  
  
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  
  refvar="semsize"
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  
  plt.title("Boxplot distribution of semantic size", size = 18)
  aoa_boxplot.set_xlabel("Semantic Size", rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title("Distribution of semsize", size = 18)
  plt.xlabel(xlabel = "Semantic Size", fontsize = 20, labelpad=20)
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  df["semsize"].describe()
  
  """The count of occurrences for this variable suggests that there are no missing values. The mean is 4.13 and the standard deviation is just above 1. This results suggests that the tendency is that is more common to think that a word has a bigger semantic size.
  
  #### Gender
  
  Gender in this dataset refers to how strongly its meaning is associated with male or female behaviour or idea. This variable could be very interesting in regards of the social bias that might, or might not, be present.
  """
  
  #calcolo frequenza assoluta per aoa
  df["gender"].value_counts()
  
  #imposto lo spazio
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  plt.rcParams["figure.figsize"] = [16, 4]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  plt.subplot(1, 2, 1)
  #distribuzione e tendenze centrali: boxplot
  gender_boxplot = sb.boxplot( x = "gender", data=df, color = colors1[1])
  plt.title("Boxplot distribution of perceived gender", size = 18)
  gender_boxplot.set_xlabel("gender", rotation = "horizontal", size = 16)
  plt.subplot(1, 2, 2)
  
  gender1= df["gender"].dropna(0, inplace = False)
  
  #imposto la legenda
  lables_size= [0-2, 2-4, 4-6, 6-8, 8-10, 10-12, 13]
  plt.legend(title="Perceived gender", title_fontsize = 14, bbox_to_anchor=(1, 1), labels=lables_size )
  #distribuzione e tendenze centrali: istogramma
  aoa_hist = plt.hist(gender1,  color = colors1[1])
  plt.title("Distribution of perceived gender", size = 18)
  plt.xlabel(xlabel = "gender", fontsize = 16)
  plt.ylabel(ylabel = "Degree", fontsize = 16)
  sb.despine(right = True)
  
  plt.show()
  
  df["gender"].describe()
  
  df["length"].describe()
  
  import matplotlib.ticker as mticker
  colors1 = ['#e5f5f9','#99d8c9','#2ca25f']
  
  refvar="gender"
  
  #BOXPLOT
  
  plt.rcParams["figure.figsize"] = [16, 8]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa_boxplot = sb.boxplot( x = refvar, data=df, color = colors1[1])
  
  #bellurie
  
  plt.title("Boxplot distribution of gender", size = 18)
  aoa_boxplot.set_xlabel("Gender", rotation = "horizontal", size = 16)
  
  
  
  #DISTRIBUTION
  plt.rcParams["figure.figsize"] = [16, 12]
  sb.set_context("notebook", font_scale=1.5, rc={"font.size":14,"axes.titlesize": 16,"axes.labelsize": 12})
  
  aoa1= df[refvar].dropna(0, inplace = False)
  
  data = aoa1
  fig, ax = plt.subplots()
  counts, bins, patches = ax.hist(data,color = colors1[1], bins=range(1,8),edgecolor='black')
  
  # Set the ticks to be at the edges of the bins.
  ax.set_xticks(bins)
  
  # Label the raw counts and the percentages below the x-axis...
  #bellurie
  plt.title("Distribution of gender", size = 18)
  plt.xlabel(xlabel = "Gender", fontsize = 20, labelpad=20)
  
  
  plt.ylabel(ylabel = "Occurences", fontsize = 16)
  
  
  ax.xaxis.set_minor_locator(mticker.FixedLocator((1, 7)))
  ax.xaxis.set_minor_formatter(mticker.FixedFormatter(("Label A", "Label B")))
  plt.setp(ax.yaxis.get_minorticklabels(), size=15, va="center")
  ax.tick_params("x",which="minor",pad=25, left=False)
  
  sb.despine(right = True)
  plt.show()
  
  aoa_descr=df[refvar].describe()
  
  print(aoa_descr.to_latex())
  
  """This variable is not self intuitive. There is no visible correlation from the number and the perceived gender of that word. A supposition is that the higher the value, the more "masculine" the word is perceived. For example the word "actor" has a perceived gender value of 5.588, where the max value is 6.971.
  
  Also it seems that the distribution is heavly centered, leaving a significant amount of outliers, as seen in the figure above.
  """
  
  df[["word","gender"]].loc[[51, 52]]
  
  """## 1.2    Distribution of variables and statistics
  
  Codice per le distribuzioni delle variabili 'arousal', 'valence', 'dominance', 'concreteness','imageability', 'familiarity'
  """
  
  df.corr() #matrice di correlazione
  
  #distribuzione delle variabili
  scaler = MinMaxScaler()
  
  var = ['arousal', 'valence', 'dominance', 'concreteness',
         'imageability', 'familiarity']
  
  for e in var:
      print(e)
      Y = df[[e]].values
      Y_minmax = scaler.fit_transform(Y)
      plt.hist(Y_minmax, edgecolor='white')
      plt.show()
  
  #piechart polisemia
  
  #percentuale delle polisemiche 
  m = 0
  p = 0
  
  for e in df['polysemy'].values:
      if e == 1:
          p += 1
      if e == 0:
          m += 1
  
  p1 = p/(p+m)*100
  m1 = m/(p+m)*100
  print(p1, m1)
  
  #grafico
  labels = 'Polisemic', 'Monosemic'
  sizes = [p1, m1]
  explode = (0.1, 0)  # only "explode" the 2nd slice (i.e. 'Hogs')
  
  fig1, ax1 = plt.subplots()
  ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
          shadow=True, startangle=90)
  ax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.
  
  plt.show()
  
  #matrice di correlazione
  corr=df.corr()
  plt.figure(figsize=(16, 6))
  heatmap = sb.heatmap(corr, vmin=-1, vmax=1, annot=True, cmap='BrBG')
  heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':18}, pad=12);
  #sb.heatmap(corr, cmap="Blues", annot=True)
  
  """### 1.2.1    Variable Normalization
  
  ## 1.3    Assessing Data Quality
  
  ### 1.3.1    Missing Values
  """
  
  df.isnull().sum()
  
  """The dataset seems to be almost without null values. In fact, there are only 14 NaN, and all are concentrated in the "web_corpus_freq" variable."""
  
  df[df['web_corpus_freq'].isnull()]
  
  """Looking at the values that are present in the dataset but are not present in the web corpus, we can spot some similarities.
  There are 3 recurrences of the same word but slightly different from one another: "Mom", "Mum" and "Mommy". Also the word "Dad" is not present in the corpus.
  
  In addition, there are 3 words written with full capital letters: "FALSE", "TRUE", and "TV". If the corpus is case sensitive, that could be an explaination of why those words are not present.
  
  Finally, there are 2 words of social media that are absent in the corpus: "Facebook" and "Twitter". This is a bit strange since those 2 words are really known.
  
  ### 1.3.2    Outliers
  """
  
  #creating a copy of the df
  df2 = df.copy()
  #dropping some variables for boxplot visualzation
  df3 = df2.drop(["word","web_corpus_freq", "polysemy","length"], axis=1)
  df3.head()
  
  #displaying all boxplot in one plot
  
  #defining outliers as a blue underscore circle
  blue_circle = dict(markerfacecolor='blue', marker='o', markeredgecolor='white')
  
  #defining fig, axs as plt.subplots. In the arguments we pass in that we want 1 row, and then the number of columns. 
  #The number of columns will be equal to the length of df.columns. 
  #Finally, we will set the figsize to 20 by 10.
  fig, axs = plt.subplots(1, len(df3.columns), figsize=(20,10))
  
  #defining the shape and color of the mean
  mean_shape = dict(markerfacecolor='green', marker='D', markeredgecolor='green')
  
  #creating a small for loop which will loop over each of our plot axes. 
  #Using the enumerate function here to keep track of i, our index value
  for i, ax in enumerate(axs.flat):
      #addong in our boxplot by calling upon ax.boxplot() and passing in df.iloc, 
      #which allows us to get columns by the index variable, i. 
      #We do this by creating square brackets, with a colon followed by a comma. 
      #This tells the iloc function to get all rows, and then we pass in the column index. 
      #Next we specify our outlier properties and pass in red_circle.
      ax.boxplot(df3.iloc[:,i], flierprops=blue_circle, showmeans = True, meanprops =mean_shape, notch=True)
      ax.set_title(df3.columns[i], fontsize=20, fontweight='bold')
      ax.tick_params(axis='y', labelsize=14)
      
  plt.tight_layout()
  
  """First, some variables had to be dropped to perform this analysys with boxplots. "word" is not a numerical variable, and for that reason is not used in this plot. "web_corpus-freq" and "polysemy" can be analised on their own.
  
  The plot shows that some variables are well distributed, like "concretness" and "imageability", while others are not, like "length", "arousal" and "dominance". 
  Later this issue will be addressed with more depth.
  """
  
  df["polysemy"].plot(kind="box", vert = False)
  
  """"polysemy" has only 2 possible values, 0 and 1, that are false and true. For that reason it doesn't have outliers."""
  
  df["polysemy"].value_counts().plot(kind="pie")
  
  """"web_corpus_freq" instead seems to have a lot of outliers, as shown below."""
  
  df["web_corpus_freq"].dropna(0, inplace = False).plot(kind="box", vert = False, logx=True)
  
  """### 1.3.3    Errors"""
  
  
  
  var = ['arousal', 'valence', 'dominance', 'concreteness',
         'imageability', 'familiarity','semsize','gender','aoa']
  
  for element in var:
    for e in df[element].values:
      if(type(e)!=np.float64):
         print(type(e))
  
    print("tutto ok fra")
  
  #creating a copy of the df
  df_errors = df.copy()
  
  df_errors[["word", "length"]]
  
  df_errors["word"] = df_errors.word.str.len()
  
  df_errors[["word", "length"]]
  
  df_errors["word"].equals(df_errors["length"])
  
  """All values of the variable 'word' are correct in regards of their length
  
  ### 1.3.4    Semantic Inconsistencies
  """
  
  df.loc[df['polysemy']==1]
  
  """## 1.4    Eventual other variable trasformations
  
  ## 1.5    Pairwise correlations and eventual elimination of variables
  
  An overview of the relation between the nine variables is provided in Fig. 6. Where a correlation greater than |0.6| is found, we plotted the values of the two variables for a better visualization (fig. 7). 
  There is a strong correlation (0.91) between concreteness and imageability: it is difficult to imagine an abstract word and easier to imagine a concrete one. Moreover, concreteness and imageability relate to the other variables similarly, with a margin of +-0.14. Therefore we merged them into a new variable, perceivability. The values of perceivabilty are the mean of concreteness and imageability values. 
  Other positively correlated variables are valence and dominance, with 0.72: the more valuable an item is perceived, the higher the degree of control over the object. 
  Familiary and age of acquisition are instead negatively related: from the pairplot (Fig \ref{fig:pairplot}) is apparent that every word acquired in early age is highly familiar.
  """
  
  #matrice di correlazione
  corr=df.corr()
  plt.figure(figsize=(16, 6))
  heatmap = sb.heatmap(corr, vmin=-1, vmax=1, annot=True, cmap='BrBG')
  heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':18}, pad=12);
  #sb.heatmap(corr, cmap="Blues", annot=True)
  
  #creating a copy of the df
  dfpp = df.copy()
  #dropping some variables for boxplot visualzation
  dfpp1 = dfpp.drop(["word", "polysemy","length", 'gender', 'web_corpus_freq', 'semsize', 'arousal'], axis=1)
  
  sb.pairplot(dfpp1, 
              plot_kws=dict(marker=".", linewidth=1),
              diag_kws=dict(fill=False),
              corner=False
             )
  
  sb.pairplot(dfpp1, 
              x_vars=['concreteness'],
              y_vars=['imageability'],
              plot_kws=dict(marker=".", linewidth=1),
              diag_kws=dict(fill=False),
              corner=False
             )
  
  #plt.savefig('pp_imageability_concreteness.png', dpi=300)
  #files.download('pp_imageability_concreteness.png')
  
  pp_dv = sb.pairplot(dfpp1, 
              x_vars=['valence'],
              y_vars=['dominance'],
              plot_kws=dict(marker=".", linewidth=1),
              diag_kws=dict(fill=False),
              corner=False
             )
  
  #plt.savefig('pp_dominance_valence.png', dpi=300)
  #files.download('pp_dominance_valence.png')
  
  sb.pairplot(dfpp1, 
              x_vars=['familiarity'],
              y_vars=['aoa'],
              plot_kws=dict(marker=".", linewidth=1),
              diag_kws=dict(fill=False),
              corner=False
             )
  
  #plt.savefig('pp_aoa_familiarity.png', dpi=300)
  #files.download('pp_aoa_familiarity.png')
  
  df2["perceivability"] = df2[["imageability", "concreteness"]].mean(axis=1)
  df2.head()
  
  df_perc=df2.drop(["concreteness","imageability"], axis=1)
  df_perc.head()
  
  
  
  """# 2    Clustering
  
  ## 2.1    Preprocessing
  """
  
  dfprepro= df_perc.copy()
  dfprepro=dfprepro.rename(columns={"gender": "masculinity"})
  
  dfprepro.loc[(dfprepro['web_corpus_freq'].isnull() == True), 'web_corpus_freq'] = dfprepro['web_corpus_freq'].mean()
  
  dfprepro[dfprepro['web_corpus_freq'].isnull()]
  
  dfprepro["web_corpus_log"] = pd.qcut(dfprepro["web_corpus_freq"], 10) #taglio la variabile web_corpus_freq in 10 gruppi
  print(dfprepro[["web_corpus_log", "web_corpus_freq"]].groupby(["web_corpus_log"], as_index=False).mean())
  
  dataframe = [dfprepro]
  
  for dataset in dataframe:
      dataset.loc[(dataset["web_corpus_freq"] > 10000) & (dataset["web_corpus_freq"] <= 100000), "web_corpus_freq"] = 4
      dataset.loc[(dataset["web_corpus_freq"] > 100000) & (dataset["web_corpus_freq"] <= 1000000), "web_corpus_freq"] = 5
      dataset.loc[(dataset["web_corpus_freq"] > 1000000) & (dataset["web_corpus_freq"] <= 10000000), "web_corpus_freq"] = 6
      dataset.loc[(dataset["web_corpus_freq"] > 10000000) & (dataset["web_corpus_freq"] <= 100000000), "web_corpus_freq"] = 7
      dataset.loc[(dataset["web_corpus_freq"] > 100000000) & (dataset["web_corpus_freq"] <= 1000000000), "web_corpus_freq"] = 8
      dataset.loc[dataset["web_corpus_freq"] > 1000000000, "web_corpus_freq"] = 9
      
  dfprepro.head()
  
  dfprepro["web_corpus_freq"].plot(kind="hist")
  
  dfprepro['web_corpus_freq'].describe()
  
  dfprepro['web_corpus_log'].describe()
  
  dfprepro = dfprepro.drop(["web_corpus_log","word"], axis=1)
  #dfprepro.loc[(dfprepro['web_corpus_freq'].isnull() == True), 'web_corpus_freq'] = dfprepro['web_corpus_freq'].mean()
  dfprepro.isnull().sum()
  dfprepro.head()
  
  pca = PCA(n_components=2)
  pca.fit(dfprepro)
  PCA_df = pd.DataFrame(pca.transform(dfprepro))
  pca.transform(dfprepro)
  score_pca = pca.transform(dfprepro)
  
  scaler = MinMaxScaler()
  
  Y_pca=PCA_df.values
  
  Y_pca_minmax = scaler.fit_transform(Y_pca)
  
  scaled_dfprepro = pd.DataFrame(data = Y_pca_minmax, columns = PCA_df.columns)
  
  scaled_dfprepro.head()
  
  X=dfprepro.values
  
  X_minmax = scaler.fit_transform(X)
  
  df_xminmax = pd.DataFrame(data = X_minmax, columns = dfprepro.columns)
  
  scaled_dfprepro.head()
  
  df_xminmax['word']=df_perc['word']
  
  cols = df_xminmax.columns.tolist()
  cols = cols[-1:] + cols[:-1]
  
  df_xminmax=df_xminmax[cols]
  
  
  df_xminmax.head()
  
  df_xminmax=df_xminmax.drop(['word'],axis=1)
  
  df_xminmax.describe()
  
  PCA_df
  
  x = PCA_df[0]
  y = PCA_df[1]
  #z = PCA_df[2]
  print(type (x))
  print(type (y))
  #print(type (z))
  print(type (Y_pca_minmax))
  
  print (Y_pca_minmax)
  
  """## 2.2    Clustering analysis by K-Means"""
  
  kmeans = KMeans( n_clusters=3, n_init=10, max_iter=100 )
  kmeans.fit(Y_pca_minmax)
  
  kmeans.labels_
  
  print('SSE', kmeans.inertia_)
  print('Silhouette', silhouette_score(Y_pca_minmax, kmeans.labels_))
  
  print('Calinski-Harabasz', metrics.calinski_harabasz_score(Y_pca_minmax, kmeans.labels_))
  
  silhouette_list = []
  
  #for k in range(2, 50):
  #    kmeans = KMeans(n_clusters=k, n_init=10, max_iter=100)
  #    kmeans.fit(Y_pca_minmax)
  #    silhouette_list.append( silhouette_score(Y_pca_minmax, kmeans.labels_))
  
  #plt.plot(range(2, len(silhouette_list)+2), silhouette_list, marker='*')
  #plt.ylabel('Silhouette score')
  #plt.show()
  
  kmeans = KMeans(n_clusters=3, n_init=10, max_iter=100)
  kmeans.fit(Y_pca_minmax)
  
  centers = kmeans.cluster_centers_
  
  kmeans.labels_
  
  from matplotlib import cm
  plt.scatter( Y_pca[:,0], Y_pca[:,1], s=40, c= kmeans.labels_ , cmap = cm.tab20c)
  
  #plt.xticks(fontsize=20)
  plt.yticks(fontsize=20)
  
  plt.show()
  
  plt.scatter( Y_pca_minmax[:,0], Y_pca_minmax[:,1], s=40, c= kmeans.labels_ , cmap = cm.tab20c)
  plt.scatter( centers[:,0], centers[:,1], c='red', marker='*', s=200 )
  
  #plt.xticks(fontsize=20)
  plt.yticks(fontsize=20)
  
  plt.show()
  
  scaled_dfprepro['cluster'] = kmeans.labels_
  
  df_xminmax['cluster'] = kmeans.labels_
  
  df_xminmax['cluster']=df_xminmax['cluster'].map({0:'1', 1:'2', 2:'3'})
  
  df_xminmax_plot=df_xminmax.drop(["polysemy"], axis=1)
  
  plt.figure(figsize = (8,8))
  pie = scaled_dfprepro.groupby(['cluster']).size().to_frame().reset_index()
  pie.rename(columns={0: 'count'}, inplace=True)
  
  pie_labels = ['first cluster', 'second cluster', 'third cluster']
  plt.pie(pie['count'], labels=pie_labels)
  plt.show()
  sb.set(font_scale=3.7)
  
  fig, axs = plt.subplots(ncols=2, nrows=6, figsize=(30,60))
  for cols, x in zip(df_xminmax_plot.columns[:5], range(5)):
      for y in range(1):
          g1=sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3'], ax=axs[x,y], showfliers = True )
          g1.set(xlabel=None)
  
  for cols, x in zip(df_xminmax_plot.columns[5:], range(5)):
      for y in range(1,2):
          g2=sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3'], ax=axs[x,y], showfliers = True )
          g2.set(xlabel=None)
  
  df_xminmax['cluster'].map({1:'1', 2:'2', 3:'3'})
  
  df_mean=df_xminmax.groupby(['cluster'],group_keys=True).median()
  
  cols = [col for col in df_mean.columns if col !="word"]
  
  fig = px.parallel_coordinates(df_mean, range_color=[0.5, 3.5],dimensions=cols, labels={"length": "length",
                  "arousal": "arousal", "valence": "valence",
                  "dominance": "dominance", "familiarity": "familiarity", "aoa": "aoa",
                  "semsize": "size", "masculinity": "masculinity",
                  "web_corpus_freq": "freq", "perceivability": "perceivability",},
                  color_continuous_scale=[(0.00, "yellow"),   (0.33, "yellow"),
                                          (0.33, "green"), (0.66, "green"),
                                          (0.66, "blue"),  (1.00, "blue")],
                                color=[1,2,3],
                               color_continuous_midpoint=2)
  fig.show()
  
  df_perc.isnull().sum()
  
  df_xminmax_plot['word']=df_perc['word']
  df_xminmax_plot.loc[df_xminmax['cluster']=='1']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='2']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='3']
  
  """## 2.3    Analysis by density-based clustering"""
  
  from sklearn.neighbors import NearestNeighbors
  
  neighbors = NearestNeighbors(n_neighbors=20)
  neighbors_fit = neighbors.fit(Y_pca_minmax)
  distances, indices = neighbors_fit.kneighbors(Y_pca_minmax)
  distances = np.sort(distances, axis=0)
  distances = distances[:,1]
  plt.plot(distances)
  
  fig = plt.figure(figsize=(20, 12))
  fig.subplots_adjust(hspace=.5, wspace=.2)
  i = 1
  
  for x in range(0, 15, 1):
      #y=-2+x/10
      eps = 0.012+1*x/1000
      db = DBSCAN(eps=eps, min_samples=20).fit(Y_pca_minmax)
      core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
      core_samples_mask[db.core_sample_indices_] = True
      labels = db.labels_
      
      #print(eps)
      ax = fig.add_subplot(3, 5, i)
      ax.title.set_text("eps = {}".format(round(eps, 3)))
      #ax.text(1, 4, "eps = {}".format(round(eps, 3)), fontsize=25, ha="center")
      sb.scatterplot(Y_pca_minmax[:,0], Y_pca_minmax[:,1], hue=["{} cluster".format(x) for x in labels],legend=False,palette='tab20c',markers='.',size=1)
      ax.text(0.5,-0.3, "{} cluster(s)".format(max(labels)+1), size=12, ha="center", transform=ax.transAxes)
  
  
      i += 1
  
  dbscan = DBSCAN( eps=0.018, min_samples=20)
  dbscan.fit(Y_pca_minmax)
  
  dbscan.labels_
  
  unique_labels, unique_counts = np.unique(dbscan.labels_, return_counts=True)
  print(max(unique_labels))
  
  #cols = [col for col in PCA_df.columns if col != 'class']
  
  plt.scatter( Y_pca_minmax[:,0], Y_pca_minmax[:,1], s=25, c= dbscan.labels_, cmap = cm.tab20 )
  
  #plt.xticks(fontsize=2)
  plt.yticks(fontsize=2)
  
  plt.show()
  
  scaled_dfprepro['cluster'] = dbscan.labels_
  
  df_xminmax['cluster'] = dbscan.labels_
  
  df_xminmax['cluster']=df_xminmax['cluster'].map({0:'First', 1:'Second', 2:'Third',3:'Fourth', 4:'Fifth', 5:'Sixth',6:'Seventh', 7:'Eighth', 8:'Nineth',9:'Tenth', 10:'Eleventh', -1:'Noise'})
  
  scaled_dfprepro['cluster'] = dbscan.labels_
  
  df_xminmax['cluster'] = dbscan.labels_
  
  df_xminmax['cluster']=df_xminmax['cluster'].map({-1:'0',0:'1', 1:'2', 2:'3',3:'4',4:'5', 5:'6',6:'7', 7:'8', 8:'9',9:'10', 10:'11'})
  
  df_xminmax_plot=df_xminmax.drop(["polysemy"], axis=1)
  
  df_xminmax_plot = df_xminmax_plot.loc[df_xminmax["cluster"] != -1]
  
  
  plt.figure(figsize = (8,8))
  
  
  pie = df_xminmax_plot.groupby(['cluster']).size().to_frame().reset_index()
  pie.rename(columns={0: 'count'}, inplace=True)
  
  pie_labels = ['noise','first cluster', 'second cluster', 'third cluster','fourth cluster','fifth cluster','sixth cluster','seventh cluster','eighth cluster','nineth cluster','tenth cluster','eleventh cluster']
  plt.pie(pie['count'], labels=pie_labels)
  plt.show()
  
  fig, axs = plt.subplots(ncols=2, nrows=6, figsize=(15,30))
  for cols, x in zip(df_xminmax_plot.columns[:5], range(5)):
      for y in range(1):
          sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3','4','5','6','7','8','9','10','11'], ax=axs[x,y], showfliers = True )
  
  for cols, x in zip(df_xminmax_plot.columns[5:], range(5)):
      for y in range(1,2):
          sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3','4','5','6','7','8','9','10','11'], ax=axs[x,y], showfliers = True )
  
  print('Silhouette', silhouette_score(Y_pca_minmax, dbscan.labels_))
  
  print('Calinski-Harabasz', metrics.calinski_harabasz_score(Y_pca_minmax, dbscan.labels_))
  
  df_perc.isnull().sum()
  
  df_xminmax_plot['word']=df_perc['word']
  df_xminmax_plot.loc[df_xminmax['cluster']=='First']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='Second']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='Third']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='Fourth']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='Fifth']
  
  """## 2.4    Analysis by hierarchical clustering"""
  
  data_dist = pdist(Y_pca_minmax, metric='euclidean')
  data_link = linkage(data_dist, method='complete')
  
  res = dendrogram(data_link, truncate_mode='lastp')
  
  hier = AgglomerativeClustering(n_clusters=6, affinity='euclidean', linkage='complete')
  hier.fit(Y_pca_minmax)
  hier.labels_
  hier.n_clusters_
  #hier.n_leaves_
  
  plt.scatter( Y_pca_minmax[:,0], Y_pca_minmax[:,1],s=50, c= hier.labels_, marker='.', cmap = cm.tab20)
  
  #plt.xticks(fontsize=20)
  plt.yticks(fontsize=20)
  
  plt.show()
  
  #Store the labels
  labels = hier.labels_
  
  #Then get the frequency count of the non-negative labels
  counts = np.bincount(labels[labels>=0])
  
  print (counts)
  
  print('Silhouette', silhouette_score(Y_pca_minmax, hier.labels_))
  
  print('Calinski-Harabasz', metrics.calinski_harabasz_score(Y_pca_minmax, hier.labels_))
  
  scaled_dfprepro['cluster'] = hier.labels_
  
  df_xminmax['cluster'] = hier.labels_
  
  df_xminmax['cluster']=df_xminmax['cluster'].map({0:'1', 1:'2', 2:'3',3:'4', 4:'5', 5:'6'})
  
  df_xminmax_plot=df_xminmax.drop(["polysemy"], axis=1)
  
  plt.figure(figsize = (8,8))
  pie = scaled_dfprepro.groupby(['cluster']).size().to_frame().reset_index()
  pie.rename(columns={0: 'count'}, inplace=True)
  
  pie_labels = ['first cluster', 'second cluster', 'third cluster','fourth cluster', 'fifth cluster', 'sixth cluster']
  plt.pie(pie['count'], labels=pie_labels)
  plt.show()
  sb.set(font_scale=3.7)
  
  
  fig, axs = plt.subplots(ncols=2, nrows=6, figsize=(30,60))
  for cols, x in zip(df_xminmax_plot.columns[:5], range(5)):
      for y in range(1):
          g1=sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3','4', '5','6'], ax=axs[x,y], showfliers = True )
          g1.set(xlabel=None)
  
  
  for cols, x in zip(df_xminmax_plot.columns[5:], range(5)):
      for y in range(1,2):
          g2=sb.boxplot(y=cols, x='cluster', data=df_xminmax_plot, order=['1', '2','3','4', '5','6'], ax=axs[x,y], showfliers = True )
          g2.set(xlabel=None)
  
  df_perc.isnull().sum()
  
  df_xminmax_plot['word']=df_perc['word']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='1']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='2']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='3']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='4']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='5']
  
  df_xminmax_plot.loc[df_xminmax['cluster']=='6']
  
  df_mean=df_xminmax.groupby(['cluster'],group_keys=True).median()
  
  cols = [col for col in df_mean.columns if col !="word"]
  
  fig = px.parallel_coordinates(df_mean, color=[1,2,3,4,5,6], range_color=[0.5, 6.5],dimensions=cols,labels={"length": "length",
                  "arousal": "arousal", "valence": "valence",
                  "dominance": "dominance", "familiarity": "familiarity", "aoa": "aoa",
                  "semsize": "size", "masculinity": "masculinity",
                  "web_corpus_freq": "frequency", "perceivability": "perceivability",},
                  color_continuous_scale=[(0.00, "yellow"),   (0.16, "yellow"),
                                          (0.17, "green"), (0.33, "green"),
                                          (0.34, "blue"),  (0.50, "blue"),
                                          (0.51, "red"),(0.67, "red"),
                                          (0.68, "pink"),  (0.83, "pink"),
                                          (0.84, "black"), (1.00, "black")
                                          ],
                               color_continuous_midpoint=5)
  fig.show()
  
  # non serve ora 
  #hier = AgglomerativeClustering(n_clusters=5, affinity='euclidean', linkage='ward')
  #hier.fit(Y_pca_minmax)
  #hier.labels_
  
  
  #fig = plt.figure()
  #ax = fig.add_subplot(111, projection='3d')
  
  #ax.scatter(Y_pca_minmax[:,0], Y_pca_minmax[:,1], Y_pca_minmax[:,2], s=5, c= hier.labels_, marker='.')
  
  
  #ax.plot3D(Yemo[:,1], Yemo[:,2], Yemo[:,3], c= hier.labels_, marker='.')
  
  #plt.scatter( Yemo[:,1], Yemo[:,2], s=50, c= hier.labels_, marker='.' )
  
  
  
  #plt.show()
  
  """## 2.5   Final Discussion"""