# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/raitharnett/ML-for-Good-Hackathon-1/blob/main/ml_for_good.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="Qv0f_vw8SH_Q"
# # ML-for-Good-Hackathon submission
# ## Team: 6
# ## Team members: 
#
# 1.   <NAME>
# 2.   <NAME>
# 3.   <NAME>
# 4.   <NAME>
# 5.   <NAME>
#
# # Overview
#
# Our approach was to investigate well known techniques to visualize the Child Mind Institute data. We focused on the description parts on a selection of the data:
#
# *   CrisisLogger: `Data/CrisisLogger/crisislogger.csv`
# *   ProlificAcademic: `Data/ProlificAcademic/updated_data/April_21/CRISIS_Adult_April_2021.csv`, `Data/ProlificAcademic/updated_data/April_21/CRISIS_Parent_April_2021.csv`
#
# We show how we used the following techniques:
#
# *   Clustering documents with TFIDF and KMeans.
# *   A word cloud generator.
# *   BERT pre-trained model to predict sentiment.
# *   Using ``spaCy`` to analyze transcriptions in the CrisisLogger data.
#
#
#
#
#
#
#

# + id="qIvHNdG8yoY_"
# %%capture
# %pip install transformers[sentencepiece] datasets torch tensorflow emoji
# %pip install --update spacy
# !python -m spacy download en_core_web_sm

# + id="-mcvivyLvj7h" colab={"base_uri": "https://localhost:8080/"} outputId="baf085ab-cce7-4cc5-e598-e0c89cee7b3f"
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
import pandas as pd
from transformers import AutoTokenizer, AutoModelForSequenceClassification, TFAutoModelForSequenceClassification, AutoModelForSeq2SeqLM
import torch
import tensorflow as tf
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.cm as cm
import seaborn as sns; sns.set()  # for plot styling

import ipywidgets as widgets
from collections import defaultdict

from sklearn.cluster import  MiniBatchKMeans, KMeans
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.datasets import make_blobs

from yellowbrick.text import TSNEVisualizer, FreqDistVisualizer
import spacy
from spacy.matcher import PhraseMatcher
import string
import nltk
import nltk.collocations as collocations

from nltk import FreqDist, bigrams
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

from wordcloud import WordCloud

import warnings
warnings.filterwarnings("ignore")
# %matplotlib inline
nltk.download('stopwords')
nltk.download('punkt')

print(spacy.__version__)  # 3.1.1

# + [markdown] id="zkCDcYd3fcrj"
# # Clustering documents with TFIDF and KMeans
# References:
# *   Visulisation details: https://www.kaggle.com/jbencina/clustering-documents-with-tfidf-and-kmeans
#
# *   SSE (Sum of Square Error): https://www.clusteranalysis4marketing.com/interpretation/sum-of-squared-error-sse/
#
# *   Plotting PCA (Principal Component Analysis): https://cran.r-project.org/web/packages/ggfortify/vignettes/plot_pca.html
#
# *   Plotting TSNE: https://towardsdatascience.com/visualising-high-dimensional-datasets-using-pca-and-t-sne-in-python-8ef87e7915b , https://medium.com/analytics-vidhya/pca-vs-t-sne-17bcd882bf3d
#
#
#

# + id="X_PR18hcfPq4"
crisisLoggerDf = pd.read_csv("/content/drive/MyDrive/ML-for-Good-Hackathon-main/Data/CrisisLogger/crisislogger.csv")
adultApril2021Df = pd.read_csv("/content/drive/MyDrive/ML-for-Good-Hackathon-main/Data/ProlificAcademic/updated_data/April_21/CRISIS_Adult_April_2021.csv")
parentApril2021Df = pd.read_csv("/content/drive/MyDrive/ML-for-Good-Hackathon-main/Data/ProlificAcademic/updated_data/April_21/CRISIS_Parent_April_2021.csv")


# + id="5BzahaQ6hblw"
# function to find optimal cluster in a range
def find_optimal_clusters(data, max_k):
    iters = range(2, max_k+1, 2)
    sse = []
    for k in iters:
        sse.append(MiniBatchKMeans(n_clusters=k, init_size=1024, batch_size=2048, random_state=20).fit(data).inertia_)
        print('Fit {} clusters'.format(k))
    # plot clusters   
    f, ax = plt.subplots(1, 1)
    ax.plot(iters, sse, marker='o')
    ax.set_xlabel('Cluster Centers')
    ax.set_xticks(iters)
    ax.set_xticklabels(iters)
    ax.set_ylabel('SSE')
    ax.set_title('SSE by Cluster Center Plot')
    plt.show(block=True)


# + colab={"base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": ["830c5e20c7d54b2baa97559cc9e7baea", "f8c4481f2907436ab803b6f4fc5ebb3b", "f4303f9f529c46be942564d7496a6b35"]} id="SPRcgGrnitQD" outputId="2f89bc9b-7e5e-49ad-d5d6-2104255175dd"
# widget specify max range
check_clusters = widgets.BoundedIntText(
 value=26,
 min=0,
 max=60,
 description='Maximum Clusters to Check:'
 )
display(check_clusters)

# + colab={"base_uri": "https://localhost:8080/", "height": 520} id="AFrew7Te4TOI" outputId="d697bded-0f9f-4691-f527-cd908314c7a1"
# compute optimal clustering for crisisLogger transcriptions
data = TfidfVectorizer(stop_words='english').fit_transform(crisisLoggerDf['transcriptions'].to_numpy())
max_custers = check_clusters.value
find_optimal_clusters(data, max_custers)


# + id="Ir5Wmrqa5fq9"
# function to TSNE (T-distributed Stochastic Neighbor Embedding)
def plot_tsne_pca(data, labels):
    max_label = max(labels)
    max_items = np.random.choice(range(data.shape[0]), size=3000, replace=True)
    
    pca = PCA(n_components=2).fit_transform(data[max_items,:].todense())
    tsne = TSNE().fit_transform(PCA(n_components=50).fit_transform(data[max_items,:].todense()))
    
    
    idx = np.random.choice(range(pca.shape[0]), size=300, replace=False)
    label_subset = labels[max_items]
    label_subset = [cm.hsv(i/max_label) for i in label_subset[idx]]
    
    f, ax = plt.subplots(1, 2, figsize=(14, 6))
    
    ax[0].scatter(pca[idx, 0], pca[idx, 1], c=label_subset)
    ax[0].set_title('PCA Cluster Plot')
    
    ax[1].scatter(tsne[idx, 0], tsne[idx, 1], c=label_subset)
    ax[1].set_title('TSNE Cluster Plot')
    plt.show(block=True)

# + colab={"base_uri": "https://localhost:8080/", "height": 49, "referenced_widgets": ["bd17ca1db7d748f8820ef7537f1d064a", "92f630a153144ceabd54db3ac3e41b72", "d2c0f006077141e5a94b44f77be1d76a"]} id="2eF0LpKX59-3" outputId="684f808b-bbee-472b-9efb-10a6831f929c"
# widget to input optimal cluster from above
cluster_count = widgets.BoundedIntText(
 value=8,
 min=0,
 max=60,
 description='Maximum Clusters to Check:'
 )
display(cluster_count)

# + colab={"base_uri": "https://localhost:8080/", "height": 389} id="mwNPr8hU6YK2" outputId="ed340104-4d9b-44c6-c26b-f74b7e43dc77"
# predicted clusters using 8 clusters and plot clustering
predictedClusters = KMeans(n_clusters=cluster_count.value, init='k-means++', max_iter=100, n_init=1).fit(data).predict(data)
plot_tsne_pca(data, predictedClusters)
# observation: no clear cluster observed!

# + [markdown] id="__uS1nm1MqaR"
# ## Word cloud and frequency plotting 

# + colab={"base_uri": "https://localhost:8080/", "height": 786} id="YRklktGR_dbB" outputId="19ae5415-43ff-4975-de6f-e8068413fff4"
def get_matching_index(labels, match):   
    return np.where(labels == match)

    
def get_cluster_text(document, indexes):   
    returnValue = ""
    for index in indexes:        
        returnValue = returnValue + document[index]
    return " ".join(returnValue)

def get_cluster_text_by_index(labels, index, document):   
    matching_indexes = get_matching_index(labels, index)
    return get_cluster_text(document, matching_indexes) 
  
def generate_word_cloud(text):   
    regular_punct = list(string.punctuation)
    regular_punct.append("'")

    stopwords = nltk.corpus.stopwords.words('english')
    stopwords.append("'m")
    stopwords.append("n't")
    stopwords.append("'s")
    stopwords.append("'re")
    
    tokens = word_tokenize(text)
    vocabulary = set(tokens)            
    tokens = [w.lower() for w in tokens if not w.lower() in stopwords and not w in regular_punct]
    frequency_dist = nltk.FreqDist(tokens)
     
    wordcloud = WordCloud()  
    wordcloud.generate_from_frequencies(frequency_dist)
    
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show(block=True)
  
def generate_word_bar(text):   
    regular_punct = list(string.punctuation)
    regular_punct.append("'")

    stopwords = nltk.corpus.stopwords.words('english')
    stopwords.append("'m")
    stopwords.append("n't")
    stopwords.append("'s")
    stopwords.append("'re")
    
    tokens = word_tokenize(text)
    vocabulary = set(tokens)            
    tokens = [w.lower() for w in tokens if not w.lower() in stopwords and not w in regular_punct]
    frequency_dist = nltk.FreqDist(tokens)
    top_20 = frequency_dist.most_common(20)
    all_fdist = pd.Series(dict(top_20))
    ngram_sorted = {k:v for k,v in sorted(all_fdist.items(), key=lambda item:item[1], reverse=True)}
    ngram_freqdist = pd.Series(ngram_sorted)
        
    fig, ax = plt.subplots(figsize=(10,10))
    bar_plot = sns.barplot(x=ngram_freqdist.values, y=ngram_freqdist.index, orient='h', ax=ax, color='#5bc0de')
    plt.title('Frequency Distribution')
    plt.show(block=True);

# Interactive function to display word cloud and bar graph of word frequency
def show_clusters_word_cloud(document, labels):
    options = ['select a cluster']
    for i in np.unique(labels):
        options.append(i)
    sel = widgets.Select(
     options=options,     
     rows=len(options),
     description='Cluster'
     )      
    output = widgets.Output()
    display(sel,output)    
    def on_value_change(change):
        with output:      
            
            if change['new'] != 'select a cluster':
                print('cluster' + str(change['new']) + ':')
                generate_word_cloud(get_cluster_text_by_index(labels,change['new'], document))
                generate_word_bar(get_cluster_text_by_index(labels,change['new'], document))                                   
    sel.observe(on_value_change, names='value')

# show_clusters_word_cloud(crisisLoggerDf['transcriptions'].to_numpy(), predictedClusters)

cluster = 4
doc = crisisLoggerDf['transcriptions'].to_numpy()
generate_word_cloud(get_cluster_text_by_index(predictedClusters, cluster, doc))
generate_word_bar(get_cluster_text_by_index(predictedClusters, cluster, doc))

# + id="HcivqIMsNoKI"


# + [markdown] id="WqbYJGFwN4qa"
# # BERT pretrained module: `rabindralamsal/finetuned-bertweet-sentiment-analysis`
# This model has been trained on English Tweets (cased) and COVID-19 English Tweets (cased) - that being so, we feel it should provide us with a good indication of the sentiment of the CrisisLogger `transcription` data and well as the `['challenges','benefits','anything_else']` descriptive columns in `Data/ProlificAcademic/updated_data/April_21/CRISIS_Parent_April_2021.csv`
#
# We also extracted phrases from the CrisisLogger `transcription` data containing what we thought would be CV-19 key words:` ["work", "family", "kids", "safe"]`. We then use our pretrained BERT model to predict the sentiment of these phrases.  

# + id="oVqk2o3tWGBG" colab={"base_uri": "https://localhost:8080/"} outputId="44c17988-ed0e-4b5e-c1e0-572cf6f666d3"
tokenizer = AutoTokenizer.from_pretrained("rabindralamsal/finetuned-bertweet-sentiment-analysis")
model = TFAutoModelForSequenceClassification.from_pretrained("rabindralamsal/finetuned-bertweet-sentiment-analysis")

# + id="2LNsW93l96ZX"
# mapping of prediction values to readable sentiment values
sentiment_map = { 0:'negative', 1:'neutral', 2:'positive', 3:'unknown'}
color_map = ['#d9534f','#5bc0de','#5cb85c', '#b8985c']
def sentiment_score(transcription):
  if (not transcription) or pd.isnull(transcription):
    return sentiment_map[3]
  tokens =  tokenizer.encode(transcription, padding=True, truncation=True, max_length=128, return_tensors="tf")
  result =  model.predict(tokens)[0]
  prediction = tf.nn.softmax(result, axis=1).numpy()
  sentiment = np.argmax(prediction)
  return sentiment_map[int(sentiment)]


# + colab={"base_uri": "https://localhost:8080/", "height": 324} id="5S3tZP4YUl3S" outputId="793074ed-baf5-43a9-a281-914c02f50001"
crisisLoggerDf['sentiment'] = crisisLoggerDf['transcriptions'].apply(lambda t: sentiment_score(t))
crisisLoggerDf.groupby('sentiment')['transcriptions'].count().plot(kind='bar',color=color_map)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="qI3dJI13WpGv" outputId="1b3f1b1c-2074-4dd1-8d63-6c52f8feb9c6"
parentApril2021DescriptiveColumns = ['challenges','benefits','anything_else']
fig, axs = plt.subplots(nrows=len(parentApril2021DescriptiveColumns),figsize=(15,15))
fig.suptitle("Sentiment value counts of descriptive columns of April_21/CRISIS_Adult_April_2021", fontsize=18)
plt.subplots_adjust(hspace=0.50)
for index, column in enumerate(parentApril2021DescriptiveColumns):
  sentiment_column = f'sentiment_{column}'
  parentApril2021Df[sentiment_column] = parentApril2021Df[column].apply(lambda t: sentiment_score(t))
  parentApril2021Df.groupby(sentiment_column)[column].count().plot(kind='bar',color=color_map, ax=axs[index], xlabel=sentiment_column, fontsize=12)

# + colab={"base_uri": "https://localhost:8080/", "height": 326} id="YjMIrlMlYChz" outputId="371aff7a-271e-4dab-c05c-fafb3fc02654"
# Focus on phrases which contain key words related to CV-19 concerns
import en_core_web_sm
nlp = en_core_web_sm.load()

# create a matcher which extracts and joins all sentances in a 
# transcription which contain the key words "work", "family", "safe", etc.
crisislogger_matcher = PhraseMatcher(nlp.vocab)
crisislogger_phrases = [nlp("work"), nlp("family"), nlp("kids"), nlp("safe")]
matched_sents = []
def on_match(matcher, doc, i, matches):
  match_id, start, end = matches[i]
  matched_sents.append(doc[start:end].sent.text)
crisislogger_matcher.add("KeyWords", crisislogger_phrases, on_match=on_match)


def crisislogger_phrases(transcription):
  doc = nlp(transcription)
  matched_sents.clear()
  crisislogger_matcher(doc)
  return ".".join(matched_sents)

# extract key word sentance and calculate sentiment - if a transcription
# contains no key works we mark it as "unknown" sentiment
crisisLoggerDf['phrases'] = crisisLoggerDf['transcriptions'].apply(lambda t: crisislogger_phrases(t))
crisisLoggerDf['sentiment_phrases'] = crisisLoggerDf['phrases'].apply(lambda t: sentiment_score(t))
crisisLoggerDf.groupby('sentiment_phrases')['transcriptions'].count().plot(kind='bar',color=color_map)

# + colab={"base_uri": "https://localhost:8080/", "height": 1000} id="rTyPzjvBgLHm" outputId="7e7f2433-a62f-4b2f-b288-887dc776f2a4"
# basic column correlation plots
corr = adultApril2021Df.corr()
fig = plt.figure(figsize=(8,8))
plt.matshow(corr, cmap='RdBu', fignum=fig.number)
plt.xticks(range(len(corr.columns)), corr.columns, rotation='vertical');
plt.yticks(range(len(corr.columns)), corr.columns);

corr_pairs = corr.unstack()
print(corr_pairs)

sorted_pairs = corr_pairs.sort_values(kind="quicksort")
sorted_pairs.head(50)

# + [markdown] id="6YvZE7LSteT-"
# # Using `spaCy` extract new insights from CrisisLogger data
# References: http://robertorocha.info/using-nlp-to-analyze-open-ended-responses-in-surveys/

# + id="XHmCEpMhva7b"
import en_core_web_sm
nlp = en_core_web_sm.load()
nlp.max_length = 1850000
doc = nlp(crisisLoggerDf['transcriptions'].str.cat(sep=' '), disable= ['ner'])

# + colab={"base_uri": "https://localhost:8080/"} id="hrNlLGHQ1FAS" outputId="e540b61a-a58c-4853-9508-1f1019f7cf07"
# overall word frequency analysis
from collections import Counter
words = [token.lemma_ for token in doc if not token.is_stop and not token.is_punct] 
word_freq = Counter(words) 
word_freq.most_common(20)

# + colab={"base_uri": "https://localhost:8080/"} id="It2UE81-5OmP" outputId="a60e2ad3-7bd6-434c-f0e2-9f9f1b0a69af"
# What are the most common adjective-noun phrases?
from spacy.matcher import Matcher 
matcher = Matcher(nlp.vocab) 
pattern = [{'POS':'ADJ'}, {'POS':'NOUN'}] 
matcher.add('ADJ_PHRASE', [pattern]) 
matches = matcher(doc) 
phrases = [] 
for match_id, start, end in matches:
    phrases.append(doc[start:end].text.lower())
    phrase_freq = Counter(phrases)
phrase_freq.most_common(30)

# + colab={"base_uri": "https://localhost:8080/"} id="fuZG3p8m6yxG" outputId="29ee623e-33b9-4e74-85f1-d1663c8455e1"
# The most common adjective that follow the phrase “I am” or “I feel”
matcher = Matcher(nlp.vocab)
pattern = [{'LOWER' : {'IN' : ['i', 'we']}}, {'OP': '?'}, 
  {'LOWER': {'IN' : ['feel', 'am', "'m", 'are', "'re"]}}, 
  {'OP': '?'}, {'OP': '?'}, {'POS':'ADJ'}]
matcher.add("FeelAdj", [pattern])
matches = matcher(doc)
feel_adj = []
for match_id, start, end in matches:
    feel_adj.extend([token.lemma_ for token in doc[start:end] if token.pos_ == 'ADJ'])
Counter(feel_adj).most_common(20)

# + colab={"base_uri": "https://localhost:8080/"} id="0l4vlacK746o" outputId="6206dede-3073-4212-dbe1-4beb5d87f382"
# Looks for phrases that start with “I/we want/need”, followed by a noun, with optional filler words in between
matcher = Matcher(nlp.vocab) 
pattern = [{'LOWER' : {'IN' : ['i', 'we']}}, {'IS_ALPHA':True, 'OP':'?'},
   {'LOWER': {'IN' : ['need', 'want']}}, {'IS_ALPHA':True, 'OP':'?'},
   {'IS_ALPHA':True, 'OP':'?'}, {'POS':'NOUN'}]
matcher.add("WantPhrase", [pattern]) 
matches = matcher(doc)
for match_id, start, end in matches:
  print(doc[start:end])

# + colab={"base_uri": "https://localhost:8080/"} id="dpRpsqrw8VME" outputId="dd1154ac-779d-4039-fd55-7f9536e3b593"
# The words that most frequently occur near the phrase “mental health”:
from spacy.matcher import PhraseMatcher
mental_health_colloc = []
matcher = PhraseMatcher(nlp.vocab, attr = 'LOWER') 
# The attr above ensures all instances are converted to lower-case so the search is case-insensitive
pattern = [nlp.make_doc('mental health')]
matcher.add('mentalHealth', pattern) 
matches = matcher(doc)
for match_id, start, end in matches:
    span = doc[start-10 : end+10]   
    mental_health_colloc.extend([token.lemma_.lower() for token in span if not token.is_stop and not token.is_punct]) 
Counter(mental_health_colloc).most_common(20)

StarcoderdataJupyterScriptsDedupFiltered