# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Cutting Semantic Corners!

# This work interests various measures:
# - Average word frequency
# - Average word concreteness
# - Average word depth in WordNet

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
from scipy import stats
from nltk.corpus import wordnet as wn

#We already have a processed file
address = "/Users/yuribizzoni/Downloads/ttr__lexdens_nouns_PLM.xlsx"
ttr_file  = pd.read_excel(address)
#ttr_file

len(ttr_file["ORG_WR_EN_lem"])

# +
#ttr_file
# -

#Concreteness - we use a dictionary
eng_co = pd.read_excel("/Users/yuribizzoni/Downloads/Translation/B7/tools/Concreteness_ratings_Brysbaert_et_al_BRM.xlsx")

# column names
list(eng_co)
#eng_co

#this dictionary contains almost 40000 different words
len(eng_co["Word"])



#We make it into a dictionary of concreteness scores
subset = eng_co[['Word', 'Conc.M']]
conc_values = dict(subset.get_values())

# +
## computing concreteness measures

categories = "ORG_WR_EN,TR_DE_EN,ORG_SP_EN,SI_DE_EN".split(",")

overlaps = []
fr_subsets = []

mean_concreteness_scores = []
mean_concreteness_scores_frequency = []
percen_very_concretes = []
percen_very_abstracts = []
con_fre_sco = []

c=0

for category in categories:
    print(category)
    subset_fr = ttr_file[[category+"_lem", "Anzahl_"+str(c)]]
    c+=1
    fr_values = dict(subset_fr.get_values())
    print(len(fr_values.keys()))
    fr_subsets.append((category,fr_values))
    
    prv, cons, fres = [],[], []
    very_con, very_abs = [],[]
    
    for w in fr_values.keys():
        if w in conc_values.keys():
            conc_score = conc_values[w]
            fre = fr_values[w]
            
            if fre>-1:
                # we add a word's conc. score proportionally to the number of times it occurs in the corpus
                # this allows us to perform a real t-test later on 
                prv+=[conc_score]*int(fre) 
                
                cons.append(conc_score)
                fres.append(fre)
            if conc_score>4: very_con.append(conc_score) # very concrete set (concreteness>4)
            if conc_score<2: very_abs.append(conc_score) # very abstract set (concreteness<2)
                
    print("Mean concreteness score: ",np.mean(cons), "Std concreteness score: ",np.std(cons))
    n_words = len(fr_values.keys())
    mean_concreteness_scores_frequency.append(np.mean(prv))
    mean_concreteness_scores.append(np.mean(cons))
    percen_very_concretes.append(len(very_con)/n_words)
    percen_very_abstracts.append(len(very_abs)/n_words)
    con_fre_sco.append(prv)
    print("\n")
# -

for i in range(4):
    print(categories[i])
    print("Percentage very abstract words: "+str(percen_very_abstracts[i]))
    print("Percentage very concrete words: "+str(percen_very_concretes[i]))
    print("Mean type concreteness score: "+str(mean_concreteness_scores[i]))
    print("Mean token concreteness score: "+str(mean_concreteness_scores_frequency[i]))
    print("\n")

# +

plt.figure(figsize=(20,10))

N = 4
OVERALL_BAR_WIDTH = .35
SENTIMENT_Y_LABEL2 = 'Percent'
SENTIMENT_TITLE2 = 'Category and concreteness'

abstra = [x for x in percen_very_abstracts]
concre = [x for x in percen_very_concretes]

labs = categories 
ind = np.arange(N)
width = OVERALL_BAR_WIDTH
#p2 = plt.bar(ind, neu, width)
p3 = plt.bar(ind, concre, width, bottom=0)
p1 = plt.bar(ind, abstra, width, bottom=concre)

plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')
#plt.legend((p1[0], p2[0], p3[0]), ('Negative', 'Neutral', 'Positive'))
plt.legend((p1[0], p3[0]), ('Very abstract (>4/5)', 'Very concrete (<2/5)'))
#plt.savefig("/Users/yuribizzoni/abstract_concrete_difference.png")
plt.show()

# +
N = len(mean_concreteness_scores_frequency)

SENTIMENT_Y_LABEL2 = "Concreteness > 3.1"
SENTIMENT_TITLE2 = 'Mean concreteness, frequency biased'

plt.figure(figsize=(15,8))

plt.bar(np.arange(N), [x-3.1 for x in mean_concreteness_scores_frequency])
plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')
#plt.yticks(ticks=[.3,.35,.4])
#plt.savefig("/Users/yuribizzoni/mean_concreteness_freqbiased.png")
plt.show()

# +
#Differences are small. Are they statistically significant?

# +
#T test comparing each concreteness distribution with each other
#(p<.05 means statistically significant)

# con_fre_sco: concreteness frequency scores for each word in a category

for array in con_fre_sco:
    for other_array in con_fre_sco:
        if array != other_array:
            print(stats.ttest_ind(array, other_array))

# +
#subset_fr

# +
#Other analyses
#subset_fr

# +
#High frequency vs low frequency words

hig_perces, low_perces = [],[]
hig_low_ratio = []

for category in categories:
    
    print(category)
    subset_fr = ttr_file[[category+"_lem", category+"_fre"]]
    fr_values = dict(subset_fr.get_values())
    high_counts = [v for v in fr_values.values() if v>=.85] # 
    all_counts = [v for v in fr_values.values() if v>-1]
    
    print("mean frequency", np.mean(all_counts))
    low_counts = [v for v in fr_values.values() if v<0.05] # 
    hig_per = len(high_counts)/len(all_counts)
    low_per =  len(low_counts)/len(all_counts)
    
    print("percentage low frequency words: ", low_per)
    
    print("percentage high frequency words: ", hig_per)
    hig_perces.append(hig_per)
    low_perces.append(low_per)
    high_low_r = len(high_counts)/len(low_counts)
    
    print("High to low frequency words ratio: ", high_low_r)
    hig_low_ratio.append(high_low_r)
    
    print("\n")

# +
SENTIMENT_Y_LABEL2 = "Ratio"
SENTIMENT_TITLE2 = 'High/Low frequency words ratio'

plt.bar(np.arange(N), [x for x in hig_low_ratio])
plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')
#plt.savefig("/Users/yuribizzoni/frequency_ratio")

plt.show()

# +
#WordNet


depth_of_frequents = []
depth_of_rares = []
avg_depth = []

depths_arrays = []


#extra = open("/Users/yuribizzoni/Downloads/B7/WN_depth_English.csv","w")

#extra.write("CORPUS\tWN_DEPTH\tWN_DEPTH_calibrated\tFREQUENCY\n")
c=0
for category in categories:
    
    subset_fr = ttr_file[[category+"_lem", category+"_fre"]]
    c+=1
    fr_values = dict(subset_fr.get_values())
    fr_subsets.append((category,fr_values))
    prv, frequents, rares = [],[],[]
    
    for w in fr_values.keys():
        w = str(w)
        syn = wn.synsets(w) # we take all synsets containing that word
        if len(syn)>0 and w!='nan':
            fre = fr_values[w] # frequency of the word
            # we take the mean depth of the synsets containing the word
            dep_ca = np.mean([s.max_depth() for s in syn][-1]) 
            tot = dep_ca 
            if fre>-1:
                # as before, we add a proportial number of scores
                # in order to compute a t-test later
                prv+=[tot]*int(fre*100) ###WE ARE USING MPF FREQ, so it is necessary to *100 or more
            if fre>=1:
                frequents+=[tot]*int(fre*100)
            if fre<.1:
                rares+=[tot]*int(fre*100)
    depths_arrays.append(frequents)
    avg_depth.append(np.mean(prv))
    depth_of_frequents.append(np.mean(frequents))
    depth_of_rares.append(np.mean(rares))

print("done")
# -

syn[0].max_depth()

depth_of_frequents

depth_of_rares



# +
SENTIMENT_Y_LABEL2 = "Mean depth (-5.5)"
SENTIMENT_TITLE2 = 'Depth of frequent words'

plt.bar(np.arange(N), [(x-5.5) for x in depth_of_frequents])
plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')

#plt.savefig("/Users/yuribizzoni/depfreq.png", bbox_inches="tight")

plt.show()

# +
SENTIMENT_Y_LABEL2 = "Mean depth (-5.5)"
SENTIMENT_TITLE2 = 'Depth of rare words'

plt.bar(np.arange(N), [x-5.5 for x in depth_of_rares])
plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')

#plt.savefig("/Users/yuribizzoni/deprare.png", bbox_inches="tight")

plt.show()

# +
SENTIMENT_Y_LABEL2 = "Mean depth (-5.5)"
SENTIMENT_TITLE2 = 'Depth of words'

plt.bar(np.arange(N), [x-5 for x in avg_depth])
plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')
plt.show()
# -

[1]

# +
##t test. This time, some contrasts are significant, others are not. 
# Some are close to the threshold (e.g. .08), others are far away.

for i in range(len(depths_arrays)):
    array = depths_arrays[i]
    for j in range(len(depths_arrays)):
        other_array = depths_arrays[j]
        if array != other_array:
            ttest = stats.ttest_ind(array, other_array)
            print(labs[i], '->', labs[j], ttest)
            if ttest[1]<.05: print("<<< Significant\n")
# -



# +
plt.figure(figsize=(20,10))

OVERALL_BAR_WIDTH = .35
SENTIMENT_Y_LABEL2 = 'Avg Depth'
SENTIMENT_TITLE2 = 'Category and WN depth'

N = len(depth_of_frequents)
avg = [x for x in avg_depth]
freq = [x for x in depth_of_frequents]
rare = [x for x in depth_of_rares]
labs = categories #sorted([x[0] for x in sent])
ind = np.arange(N)
width = OVERALL_BAR_WIDTH

p3 = plt.bar(ind, freq, width, bottom=rare)
p1 = plt.bar(ind, rare, width, bottom=avg)
p2 = plt.bar(ind, avg, width, bottom=0)


plt.ylabel(SENTIMENT_Y_LABEL2)
plt.title(SENTIMENT_TITLE2)
plt.xticks(ind, labs, rotation='vertical')
plt.legend((p1[0], p2[0], p3[0]), ('Rare', 'Average', 'Frequent'))
#plt.legend((p1[0], p3[0]), ('Very frequent (>.9)', 'Average'))
#plt.savefig("/Users/yuribizzoni/generality_per_frequency_class.png")
plt.show()
# -

[round(x,3) for x in freq]



StarcoderdataJupyterScriptsDedupFiltered