# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Step 3: Pattern Confirmation
#
# 1. [Analysis 1: Specificty and Concreteness](#a1)
#     2. [Measuring Specificity using WordNet](#spec)
#     2. [Measuring Concreteness using a Crowdsourced Weighted Dictionary](#concrete)<br><br> 
# 3. [Analysis 2: Counting People and Organizations Mentioned using Named Entity Recognition](#ner)

# <a id='a1'></a>
# # Analysis 1: Specificty and Concreteness

import pandas
import nltk
from nltk import word_tokenize
from nltk.corpus import wordnet as wn
from nltk.corpus import stopwords
import numpy as np
import scipy
import matplotlib.pyplot as plt
import json


#Define function to count the number of hypernyms for each noun and verb
def specificty(x):
    x = x.replace('[\x00-\x1f]'," ")
    text = word_tokenize(x)
    total_list = []
    for w in text:
        if not wn.synsets(w):
            pass
        else:
            synset = wn.synsets(w)
            #limit to nouns and verbs, as other words are not arranged hierarchically
            if ((synset[0].pos() == (wn.NOUN)) or (synset[0].pos() == (wn.VERB))):
                #I assume the most popular definition of each word.
                paths = synset[0].hypernym_paths()
                a_path = []
                for num in range(0,len(paths)):
                    a_path.append(len([synset.name for synset in paths[num]]))
                    #I am taking the path with the minimum number of hypernyms, but this could be calculated some other way.
                    path_num = min(a_path)
                total_list.append( (w, path_num) )
    return total_list



#Function to calculate the concreteness of a word based on the crowdsourced dictionary
def concrete(x, dict):
    x = x.replace('[\x00-\x1f]'," ")
    x = x.lower()
    text = word_tokenize(x)
    text = [word for word in text if word not in stopwords.words('english')]
    concrete_score = []
    for w in text:
        if w in dict:
            concrete_score.append((w, dict[w]))
    return concrete_score


# ### Create Strings

# +
#first create strings for the two comparison texts,
#Kant's The Metaphysical Elements of Ethics
#and the Wikipedia page on Germany

kant_string = open("../input_data/kant_metaphysics.txt", 'r', encoding='utf-8').read()
wiki_string = open("../input_data/wiki_germany.txt", 'r', encoding='utf-8').read()
# -

#Read in our dataframe to extract text
df = pandas.read_csv("../data/comparativewomensmovement_dataset.csv", sep='\t', index_col=0, encoding='utf-8')
df

# +
#concatenate the documents from each organization together, creaing four strings

redstockings = df[df['org']=='redstockings']
redstockings_string = ' '.join(str(s) for s in redstockings['text_string'].tolist())
cwlu = df[df['org']=='cwlu']
cwlu_string = ' '.join(str(s) for s in cwlu['text_string'].tolist())
heterodoxy = df[df['org']=='heterodoxy']
heterodoxy_string = ' '.join(str(s) for s in heterodoxy['text_string'].tolist())
hullhouse = df[df['org']=='hullhouse']
hullhouse_string = ' '.join(str(s) for s in hullhouse['text_string'].tolist())
# -

# <a id='spec'></a>
# ## Specificity Score

# +
#Calculate specificity score for each noun and verb in each string
#Creates a list with word and specificity score for each string

kant_specificity = specificty(kant_string)
wiki_specificity = specificty(wiki_string)
redstockings_specificity = specificty(redstockings_string)
cwlu_specificity = specificty(cwlu_string)
heterodoxy_specificity = specificty(heterodoxy_string)
hullhouse_specificity = specificty(hullhouse_string)
# -

#extract just the specificity score from each list
kant_specificity_array = list(int(x[1]) for x in kant_specificity)
wiki_specificity_array = list(int(x[1]) for x in wiki_specificity)
cwlu_specificity_array = list(int(x[1]) for x in cwlu_specificity)
heterodoxy_specificity_array = list(int(x[1]) for x in heterodoxy_specificity)
hh_specificity_array = list(int(x[1]) for x in hullhouse_specificity)
red_specificity_array = list(int(x[1]) for x in redstockings_specificity)

# +
#check for a normal distribution
fig = plt.figure()
ax1 = fig.add_subplot(221)   #top left
ax2 = fig.add_subplot(222)   #top right
ax3 = fig.add_subplot(223)   #bottom left
ax4 = fig.add_subplot(224)   #bottom right

ax1.hist(heterodoxy_specificity_array, bins=10)
ax1.set_title("Heterodoxy")
ax2.hist(hh_specificity_array, bins = 10)
ax2.set_title("Hull House")
ax3.hist(red_specificity_array, bins = 10)
ax3.set_title("Redstockings")
ax4.hist(cwlu_specificity_array, bins = 10)
ax4.set_title("CWLU")

plt.tight_layout()
plt.show()
# -

# ### Compare the Distributions

# +
#print descriptive stats

print("Mean Specificity Score for Kant")
print(np.mean(kant_specificity_array))
print("Mean Specificity Score for Wikipedia entry on Germany")
print(np.mean(wiki_specificity_array))
print("Mean Specificity Score for Heterodoxy")
print(np.mean(heterodoxy_specificity_array))
print("Mean Specificity Score for Hull House")
print(np.mean(hh_specificity_array))
print("Mean Specificity Score for Redstockings")
print(np.mean(red_specificity_array))
print("Mean Specificity Score for CWLU")
print(np.mean(cwlu_specificity_array))

# +
#create an array for each city, and an array for each wave, for comparrison
newyork_specificity_array = red_specificity_array + heterodoxy_specificity_array
chicago_specificity_array = cwlu_specificity_array + hh_specificity_array

firstwave_specificity_array = hh_specificity_array + heterodoxy_specificity_array
secondwave_specificity_array = cwlu_specificity_array + red_specificity_array
# -

#compare percent difference on the specificity scale (1:18) for the test arrays
(np.mean(wiki_specificity_array) - np.mean(kant_specificity_array)) / (max(wiki_specificity_array) - min(kant_specificity_array))

#compare percent difference on the specificity scale (1:18) for the city arrays
(np.mean(chicago_specificity_array) - np.mean(newyork_specificity_array)) / (max(chicago_specificity_array) - min(newyork_specificity_array))

#compare percent difference on the specificity scale (1:18) for the wave arrays
#note this difference is much smaller than the city-based difference
(np.mean(firstwave_specificity_array) - np.mean(secondwave_specificity_array)) / (max(firstwave_specificity_array) - min(secondwave_specificity_array))

#calculate ttest statistics on city and wave arrays
#note the statistic is much smaller on the wave-based arrays compared to the city-based arrays
print(scipy.stats.ttest_ind(chicago_specificity_array, newyork_specificity_array))
print(scipy.stats.ttest_ind(firstwave_specificity_array, secondwave_specificity_array))

# <a id='concrete'></a>
# ## Concreteness Score

#Read in the dictionary created by Brysbaert et al.
dict_df = pandas.read_excel("../input_data/Concreteness_ratings_Brysbaert_et_al_BRM.xlsx",sheetname="Sheet1")
dict_df = dict_df[dict_df['Bigram']==0]
word_dict = dict_df.set_index("Word")['Conc.M'].to_dict()

# +
#Calculate concreteness score for each noun and verb in each string
#Creates a list of tuples, with word and concreteness score for each string

kant_concrete = concrete(kant_string, word_dict)
wiki_concrete = concrete(wiki_string, word_dict)
redstockings_concrete = concrete(redstockings_string, word_dict)
cwlu_concrete = concrete(cwlu_string, word_dict)
heterodoxy_concrete = concrete(heterodoxy_string, word_dict)
hullhouse_concrete = concrete(hullhouse_string, word_dict)
# -

#extract just the concreteness score from each list
kant_concrete_array = list(int(x[1]) for x in kant_concrete)
wiki_concrete_array = list(int(x[1]) for x in wiki_concrete)
cwlu_concrete_array = list(int(x[1]) for x in cwlu_concrete)
heterodoxy_concrete_array = list(int(x[1]) for x in heterodoxy_concrete)
hh_concrete_array = list(int(x[1]) for x in hullhouse_concrete)
red_concrete_array = list(int(x[1]) for x in redstockings_concrete)

# ### Compare the Distributions

# +
#check for a normal distribution
fig2 = plt.figure()
ax1 = fig2.add_subplot(221)   #top left
ax2 = fig2.add_subplot(222)   #top right
ax3 = fig2.add_subplot(223)   #bottom left
ax4 = fig2.add_subplot(224)   #bottom right

ax1.hist(heterodoxy_concrete_array, bins=5)
ax1.set_title("Heterodoxy")
ax2.hist(hh_concrete_array, bins = 5)
ax2.set_title("Hull House")
ax3.hist(red_concrete_array, bins = 5)
ax3.set_title("Redstockings")
ax4.hist(cwlu_concrete_array, bins = 5)
ax4.set_title("CWLU")

plt.tight_layout()
plt.show()

# +
#print descriptive stats

print("Mean Concreteness Score for Kant")
print(np.mean(kant_concrete_array))
print("Mean Concreteness Score for Wikipedia entry on Germany")
print(np.mean(wiki_concrete_array))
print("Mean Concreteness Score for Heterodoxy")
print(np.mean(heterodoxy_concrete_array))
print("Mean Concreteness Score for Hull House")
print(np.mean(hh_concrete_array))
print("Mean Concreteness Score for Redstockings")
print(np.mean(red_concrete_array))
print("Mean Concreteness Score for CWLU")
print(np.mean(cwlu_concrete_array))

# +
#create one array for each city
newyork_concrete_array = heterodoxy_concrete_array + red_concrete_array
chicago_concrete_array = hh_concrete_array + cwlu_concrete_array

#create one array for each wave
firstwave_concrete_array = heterodoxy_concrete_array + hh_concrete_array
secondwave_concrete_array = red_concrete_array + cwlu_concrete_array
# -

#compare percent difference on the concreteness scale (1:5) for the test arrays
(np.mean(wiki_concrete_array) - np.mean(kant_concrete_array)) / (5-1)

#compare percent difference on the concreteness scale (1:5) for the city-based arrays
(np.mean(chicago_concrete_array) - np.mean(newyork_concrete_array)) / (5-1)

#compare percent difference on the concreteness scale (1:5) for the wave-based arrays
#notice this percent difference is around half as much as the city-based differernce
(np.mean(firstwave_concrete_array) - np.mean(secondwave_concrete_array)) / (5-1)

#calculate ttest statistics on the city- and wave-based arrays
#note the statistic is more than twice as large for the New York/Chicago comparison versus the first wave/second wave comparison
print(scipy.stats.ttest_ind(newyork_concrete_array, chicago_concrete_array))
print(scipy.stats.ttest_ind(firstwave_concrete_array, secondwave_concrete_array))

# <a id='ner'></a>
# # Analysis 2: Count Organizations and People Mentioned using NER

# The below code counts the number of persons and organizations mentioned, and compares across organizations.
#
# **Note:** Because the published data is sorted, and not the full text, the code below will not reproduce the actual named entities counted. Instead, I will read in the saved named entities, count them, and print the output.

# +
#############################################################################################
##Don't run this code to reproduce the named entities. It will not work on the sorted text.##
#############################################################################################


def extract_entities(text):
    #text = text.decode('ascii','ignore') #convert all characters to ascii
    text  = re.sub('[\x00-\x1f]'," ", text)
    org_list = []
    person_list = []

    for sent in nltk.sent_tokenize(text):
        chunked = nltk.ne_chunk(nltk.pos_tag(nltk.word_tokenize(sent)))
        for n in chunked:
            if isinstance(n, nltk.tree.Tree):
                if n.label()=="ORGANIZATION":
                    org_list.append(' '.join(c[0] for c in n.leaves()))
                if n.label()=="PERSON":
                    person_list.append(' '.join(c[0] for c in n.leaves()))
    return org_list, person_list

org_strings = [hullhouse_string, heterodoxy_string, cwlu_string, redstockings_string]


for org in org_strings:
    org_list, person_list = extract_entities(org)
    myList = [org_list, person_list]
    filename="../input_data/named_entities_%s.json" % org
    
    ##Uncomment the line below to save the named entities as a JSON file
    #json.dump( myList_cwlu, open( filename, "w", endoding = 'utf-8' ) )


# +
##################################################################
##Count saved named entities to reproduce named entity analysis.##
##################################################################

myList_cwlu = json.load( open ("../input_data/named_entities_cwlu.json", "r", 
                              encoding='utf-8') )

cwlu_orgs =  myList_cwlu[0]
cwlu_person = myList_cwlu[1]


myList_hh = json.load( open ("../input_data/named_entities_hullhouse.json", "r", 
                              encoding = 'utf-8') )

hh_orgs =  myList_hh[0]
hh_person = myList_hh[1]


myList_red = json.load( open ("../input_data/named_entities_redstockings.json", "r"),
                     encoding='utf-8')

red_orgs =  myList_red[0]
red_person = myList_red[1]


myList_heterodoxy = json.load( open ("../input_data/named_entities_heterodoxy.json", "r"),
                            encoding='utf-8')

heterodoxy_orgs =  myList_heterodoxy[0]
heterodoxy_person = myList_heterodoxy[1]

# +
#plots the number of organizations and persons mentioned by each organization

import matplotlib.pyplot as plt
 
# data to plot
n_groups = 4
num_persons = (len(hh_person), len(heterodoxy_person), len(cwlu_person), len(red_person))
num_orgs = (len(hh_orgs), len(heterodoxy_orgs), len(cwlu_orgs), len(red_orgs))

 
# create plot
fig, ax = plt.subplots()
index = np.arange(n_groups)
bar_width = 0.35
opacity = 0.8
 
counts1 = plt.bar(index, num_persons, bar_width,
                 alpha=opacity,
                 color='b',
                 label='Persons')
 
counts2 = plt.bar(index + bar_width, num_orgs, bar_width,
                 alpha=opacity,
                 color='g',
                 label='Organizations')
 
plt.xlabel('Organization')
plt.ylabel('Count of Named Entities')
plt.title('Count of Named Entities by Organization')
plt.xticks(index + bar_width, ('Hull House', 'Heterodoxy', 'CWLU', 'Redstockings'))
plt.legend(loc='upper left')
 
plt.tight_layout()
plt.show()

StarcoderdataJupyterScriptsDedupFiltered