# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + [markdown] id="view-in-github" colab_type="text"
# <a href="https://colab.research.google.com/github/teampurpler/Mechine_Learning_Projects/blob/main/Unsupervised_Learning_Project__Document_Clustering_and_Topic_Modeling.ipynb" target="_parent"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/></a>

# + [markdown] id="Xp3LTq59E8UK"
# # Document Clustering and Topic Modeling 

# + [markdown] id="mS-ob0Q8E8UM"
# *In* this project, we use unsupervised learning models to cluster unlabeled documents into different groups, visualize the results and identify their latent topics/structures.

# + [markdown] id="ZE7Qr-ccE8UN"
# ## Contents

# + [markdown] id="2S0waFW9E8UO"
# * [Part 1: Load Data](#Part-1:-Load-Data)
# * [Part 2: Tokenizing and Stemming](#Part-2:-Tokenizing-and-Stemming)
# * [Part 3: TF-IDF](#Part-3:-TF-IDF)
# * [Part 4: K-means clustering](#Part-4:-K-means-clustering)
# * [Part 5: Topic Modeling - Latent Dirichlet Allocation](#Part-5:-Topic-Modeling---Latent-Dirichlet-Allocation)
#

# + [markdown] id="5nXGAelMjJFq"
# # Part 0: Setup Google Drive Environment

# + id="6eT1n7oijJ8v"
# !pip install -U -q PyDrive

from pydrive.auth import GoogleAuth
from pydrive.drive import GoogleDrive
from google.colab import auth
from oauth2client.client import GoogleCredentials

auth.authenticate_user()
gauth = GoogleAuth()
gauth.credentials = GoogleCredentials.get_application_default()
drive = GoogleDrive(gauth)

# + id="0N2bb9S6jKGh"
file = drive.CreateFile({'id':'192JMR7SIqoa14vrs7Z9BXO3iK89pimJL'}) # replace the id with id of file you want to access
file.GetContentFile('data.tsv')  # tab-separated

# + [markdown] id="Nc9DK62BE8UP"
# # Part 1: Load Data

# + id="OjdBV8gGE8UQ" colab={"base_uri": "https://localhost:8080/"} outputId="2013fe14-94e9-4665-d8d5-f58625985984"
import numpy as np
import pandas as pd
import nltk
# import gensim, nltk = natural language toolkit

from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt

nltk.download('punkt')
nltk.download('stopwords')

# + id="sIj2Z8T70FZD" colab={"base_uri": "https://localhost:8080/"} outputId="5ea4bf67-fd3e-488c-e69b-01dbf9d68e32"
# Load data into dataframe
df = pd.read_csv('data.tsv', sep='\t', error_bad_lines=False)

# + id="oVIWPym-8MnJ" colab={"base_uri": "https://localhost:8080/", "height": 547} outputId="7f537cbc-1a0b-4caf-cd97-382b2a40e1bb"
df.head()

# + id="aoRSRJzZA7eT"
# Remove missing value
df.dropna(subset=['review_body'],inplace=True)

# + id="bbRq4wqlpV_s"
df.reset_index(inplace=True, drop=True)

# + id="oDJY4z3qoB77" colab={"base_uri": "https://localhost:8080/"} outputId="73f62eed-fec6-41ff-e70a-5fc4d3d743fe"
df.info()

# + id="VbKDogiW79a6"
# use the first 1000 data as our training data
data = df.loc[:999, 'review_body'].tolist()

# + id="PCp4l6u2zj9b" colab={"base_uri": "https://localhost:8080/"} outputId="34d13937-8c2d-4a09-c2b1-721b0b1d29f8"
data

# + [markdown] id="iQ4KGnVeE8UX"
# # Part 2: Tokenizing and Stemming

# + [markdown] id="GHHIaFATE8UY"
# Load stopwords and stemmer function from NLTK library.
# Stop words are words like "a", "the", or "in" which don't convey significant meaning.
# Stemming is the process of breaking a word down into its root.

# + id="3gSwiUBRE8UY" colab={"base_uri": "https://localhost:8080/"} outputId="a8ee3605-609f-45aa-ed4d-e48d1d0bd36e"
# Use nltk's English stopwords.
stopwords = nltk.corpus.stopwords.words('english') #stopwords.append("n't")
stopwords.append("'s")
stopwords.append("'m")
stopwords.append("br") #html <br>
stopwords.append("watch")

print ("We use " + str(len(stopwords)) + " stop-words from nltk library.")
print (stopwords[:10])

# + [markdown] id="WtDXMCeME8Uh"
# Use our defined functions to analyze (i.e. tokenize, stem) our reviews.

# + id="e50130X8E8Uc"
from nltk.stem.snowball import SnowballStemmer
# from nltk.stem import WordNetLemmatizer 

stemmer = SnowballStemmer("english")

# tokenization and stemming
def tokenization_and_stemming(text):
    tokens = []
    # exclude stop words and tokenize the document, generate a list of string 
    for word in nltk.word_tokenize(text):
        if word.lower() not in stopwords:
            tokens.append(word.lower())

    filtered_tokens = []
    
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    for token in tokens:
        if token.isalpha():
            filtered_tokens.append(token)
            
    # stemming
    stems = [stemmer.stem(t) for t in filtered_tokens]
    return stems


# + id="ywzEhcMIN5K6" colab={"base_uri": "https://localhost:8080/"} outputId="979e8889-db9c-48ef-fdc5-6c8ea788d06c"
tokenization_and_stemming(data[0])

# + id="OpVJjUke4YYp" colab={"base_uri": "https://localhost:8080/", "height": 35} outputId="080d1efa-ce9e-478d-926f-5bfcd1f58306"
data[0]

# + [markdown] id="QAWdFqL5E8Uo"
# # Part 3: TF-IDF
#
# TF: Term Frequency
#
# IDF: Inverse Document Frequency

# + id="k-XH7R4pE8Up" colab={"base_uri": "https://localhost:8080/"} outputId="04a89424-0dab-4a8d-afbd-5be23b902723"
from sklearn.feature_extraction.text import TfidfVectorizer
# define vectorizer parameters
# TfidfVectorizer will help us to create tf-idf matrix
# max_df : maximum document frequency for the given word
# min_df : minimum document frequency for the given word
# max_features: maximum number of words
# use_idf: if not true, we only calculate tf
# stop_words : built-in stop words
# tokenizer: how to tokenize the document
# ngram_range: (min_value, max_value), eg. (1, 3) means the result will include 1-gram, 2-gram, 3-gram
tfidf_model = TfidfVectorizer(max_df=0.99, max_features=1000,
                                 min_df=0.01, stop_words='english',
                                 use_idf=True, tokenizer=tokenization_and_stemming, ngram_range=(1,1))

tfidf_matrix = tfidf_model.fit_transform(data) #fit the vectorizer to synopses

print ("In total, there are " + str(tfidf_matrix.shape[0]) + \
      " reviews and " + str(tfidf_matrix.shape[1]) + " terms.")

# + colab={"base_uri": "https://localhost:8080/"} id="1VpryOuvM9jV" outputId="cf00d4fc-3631-4306-e193-f55e4d260435"
tfidf_matrix

# + id="y1JYAhxDKmmE" colab={"base_uri": "https://localhost:8080/"} outputId="48a93a1d-9aac-4791-87ad-cb61070ac5c8"
tfidf_matrix.toarray() #todense()

# + colab={"base_uri": "https://localhost:8080/"} id="oYaO5bRhCmyC" outputId="f59ca303-6c3d-4f56-fbc4-3f16faeac206"
tfidf_matrix.todense()

# + colab={"base_uri": "https://localhost:8080/"} id="UTGoc6KkCZ4Q" outputId="a4f1f7b7-954e-417b-899b-aca2fd4487d7"
print(type(tfidf_matrix.toarray()))

# + colab={"base_uri": "https://localhost:8080/"} id="PEn9UdG1Celo" outputId="2a076633-8cb2-45e0-8479-bfcee7930f69"
print(type(tfidf_matrix.todense()))

# + [markdown] id="whu1pCOiE8Uv"
# Save the terms identified by TF-IDF.

# + id="lLdKk6n-E8Uw"
# words
tf_selected_words = tfidf_model.get_feature_names()

# + id="6mWvzNWQFB26" colab={"base_uri": "https://localhost:8080/"} outputId="57ded01a-9178-4595-a503-89bb09ee1d97"
# print out words
tf_selected_words

# + [markdown] id="XEcwtws5E8U8"
# # Part 4: K-means clustering

# + id="7LJQ5i3IE8U9"
# k-means clustering
from sklearn.cluster import KMeans

num_clusters = 5

# number of clusters
km = KMeans(n_clusters=num_clusters)
km.fit(tfidf_matrix)

clusters = km.labels_.tolist()

# + [markdown] id="fboVpRAfE8U-"
# ## 4.1. Analyze K-means Result

# + id="KGs4aIIME8U_"
# create DataFrame films from all of the input files.
product = { 'review': df[:1000].review_body, 'cluster': clusters}
frame = pd.DataFrame(product, columns = ['review', 'cluster'])

# + id="APmEUmm6E8VC" colab={"base_uri": "https://localhost:8080/", "height": 359} outputId="6be6cbc0-44ad-4373-de07-32aa70d1b834"
frame.head(10)

# + id="Ht1SbbOSE8VE" colab={"base_uri": "https://localhost:8080/", "height": 221} outputId="6f2c76b6-ca37-449a-b424-b9dca197ef4d"
print ("Number of reviews included in each cluster:")
frame['cluster'].value_counts().to_frame()

# + id="phDfUdh5qL9G" colab={"base_uri": "https://localhost:8080/"} outputId="13810159-6c8a-403a-ed91-60c345bc8684"
km.cluster_centers_

# 239数的list -> cluster 0的中心点的tf-idf值
#-> assumption: 中心点的值可以代表这个cluster
#-> tf-idf值越大，对应的词越能代表这个document
#-> 选出了tf-idf最大的6个值对应的词来代表这个cluster

# + id="vrTpBeYFuIn5" colab={"base_uri": "https://localhost:8080/"} outputId="e225f691-5b1a-4710-fe0a-ca66b2075db3"
km.cluster_centers_.shape

# + id="tQg0CF11chKK" colab={"base_uri": "https://localhost:8080/"} outputId="9c4788b6-e293-46d0-ee7f-532b571d591a"
print ("<Document clustering result by K-means>")

#km.cluster_centers_ denotes the importances of each items in centroid.
#We need to sort it in decreasing-order and get the top k items.
order_centroids = km.cluster_centers_.argsort()[:, ::-1] 

Cluster_keywords_summary = {}
for i in range(num_clusters):
    print ("Cluster " + str(i) + " words:", end='')
    Cluster_keywords_summary[i] = []
    for ind in order_centroids[i, :6]: #replace 6 with n words per cluster
        Cluster_keywords_summary[i].append(tf_selected_words[ind])
        print (tf_selected_words[ind] + ",", end='')
    print ()
    
    cluster_reviews = frame[frame.cluster==i].review.tolist()
    print ("Cluster " + str(i) + " reviews (" + str(len(cluster_reviews)) + " reviews): ")
    print (", ".join(cluster_reviews))
    print ()

# + [markdown] id="oYOZXL53E8VV"
# # Part 5: Topic Modeling - Latent Dirichlet Allocation

# + id="lBPVbFNFE8VW"
# Use LDA for clustering
from sklearn.decomposition import LatentDirichletAllocation
lda = LatentDirichletAllocation(n_components=5)

# + id="9PU80SW391II" colab={"base_uri": "https://localhost:8080/"} outputId="7d1bec4c-db59-452b-df0a-6d1d971b651f"
# document topic matrix for tfidf_matrix_lda
lda_output = lda.fit_transform(tfidf_matrix)
print(lda_output.shape)
print(lda_output)

# + id="hCJs3lBz90vY" colab={"base_uri": "https://localhost:8080/"} outputId="4aa0234b-8886-44a6-8705-bfc5cb20a03e"
# topics and words matrix
topic_word = lda.components_
print(topic_word.shape)
print(topic_word)

# + id="xCEVfJS5AEgx" colab={"base_uri": "https://localhost:8080/", "height": 359} outputId="121e55f7-183a-4778-9651-7ba18fc13729"
# column names
topic_names = ["Topic" + str(i) for i in range(lda.n_components)]

# index names
doc_names = ["Doc" + str(i) for i in range(len(data))]

df_document_topic = pd.DataFrame(np.round(lda_output, 2), columns=topic_names, index=doc_names)

# get dominant topic for each document
topic = np.argmax(df_document_topic.values, axis=1)
df_document_topic['topic'] = topic

df_document_topic.head(10)

# + id="InPLDW7kBSOc" colab={"base_uri": "https://localhost:8080/", "height": 204} outputId="af2982a4-87ca-4d3d-f1e8-8085910daa8b"
df_document_topic['topic'].value_counts().to_frame()

# + id="_yLe_RFHCz0a" colab={"base_uri": "https://localhost:8080/", "height": 338} outputId="d85f51fd-f17b-4988-f2a6-7fe063765340"
# topic word matrix
print(lda.components_)
# topic-word matrix
df_topic_words = pd.DataFrame(lda.components_)

# column and index
df_topic_words.columns = tfidf_model.get_feature_names()
df_topic_words.index = topic_names

df_topic_words.head()


# + id="gbU9U8V-DFDX" colab={"base_uri": "https://localhost:8080/", "height": 204} outputId="f9c2697c-1097-475f-8094-180e92fcccb0"
# print top n keywords for each topic
def print_topic_words(tfidf_model, lda_model, n_words):
    words = np.array(tfidf_model.get_feature_names())
    topic_words = []
    # for each topic, we have words weight
    for topic_words_weights in lda_model.components_:
        top_words = topic_words_weights.argsort()[::-1][:n_words]
        topic_words.append(words.take(top_words))
    return topic_words

topic_keywords = print_topic_words(tfidf_model=tfidf_model, lda_model=lda, n_words=15)        

df_topic_words = pd.DataFrame(topic_keywords)
df_topic_words.columns = ['Word '+str(i) for i in range(df_topic_words.shape[1])]
df_topic_words.index = ['Topic '+str(i) for i in range(df_topic_words.shape[0])]
df_topic_words

StarcoderdataJupyterScriptsDedupFiltered