# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Text mining - Clustering
#
# Machine Learning types:
# - Supervised learing (labeled data),
# - Unsupervised learning (not labeled data),
# - Semi-supervised learning (somewhere in the middle).
#
# In this notebook we:
# - Scrape all quotes (save both all and only the first page),
# - Vectorize quotes using TF-IDF vectorizer,
#     - TF: Term frequency = how frequently a term appears in the target observation (quote),
#     - IDF: Inverce document frequency = is that word unique to that selected observation (quote or not).
# - Use vectorized words to cluster all the quotes using:
#     - k-means clustering: unsupervised learning methods, that calculates distance between vectors and determines quotes that are "close" to each other based on some similarity metric (i.e. Euclidian distance). Number of clusters predetermined.
#     - hiearchical (agglomerative) clustering: starts with single word clusters (bottom up approach) and merges simjilar words until forms a single cluster for the total input. The biggest hierarchical distance determines number of clusters.
# - Use quotes to tokenize them (just splitting by space for simplicity) and calculating word vectors to receive similar words (uses Neural Networks, is considered semi-supervised approach).

# +
import time
import requests
import numpy as np
import pandas as pd
from itertools import chain
from bs4 import BeautifulSoup
import matplotlib.pyplot as plt

from textblob import TextBlob
from gensim.models import word2vec
from scipy.cluster.hierarchy import ward, dendrogram

from sklearn.metrics.pairwise import cosine_similarity
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans, AgglomerativeClustering


# -

# ### Scraping

# +
def get_quotes(url):
    page = BeautifulSoup(requests.get(url).content, "html.parser")
    quotes = [i.get_text() for i in page.find_all("span",class_="text")]
    time.sleep(3)
    return quotes

quotes = get_quotes("http://quotes.toscrape.com/")

urls = ["http://quotes.toscrape.com/page/"+str(i)+"/" for i in range(1,11)]
quotes_all = [get_quotes(i) for i in urls]
quotes_all = chain.from_iterable(quotes_all)
# -

# ### TF-IDF vectorization

tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(quotes)
print(tfidf_matrix.shape)

features = tfidf_vectorizer.get_feature_names()
data = tfidf_matrix.toarray()
tfidf_df = pd.DataFrame(data,columns=features)

# ### K-Means clustering

k=5
k5 = KMeans(n_clusters=k)
k5.fit(tfidf_matrix)
clusters = k5.labels_.tolist()
my_dict = {'quotes': quotes, 'cluster': clusters}
df = pd.DataFrame(my_dict)
print(df)
df.cluster.value_counts()

# ### Important terms according to K-Means

# +
important_terms = k5.cluster_centers_.argsort()[:, ::-1]
key_list = list(tfidf_vectorizer.vocabulary_.keys())
val_list = list(tfidf_vectorizer.vocabulary_.values())
key_list[val_list.index(74)]

for i in range(k):
    for j in important_terms[i, :5]:
        print("Cluster: ", i, key_list[val_list.index(j)])
# -

# ### Hierarchical (Agglomerative) clustering

# +
dist = 1 - cosine_similarity(tfidf_matrix)
linkage_matrix = ward(dist)

plt.subplots(figsize=(15, 20))
dendrogram(linkage_matrix, orientation="right", labels=quotes)

plt.savefig('clusters.png')
# -

# ### Gensim - Word2Vec

tokenized_sentences = [sentence.split() for sentence in quotes_all]
model = word2vec.Word2Vec(tokenized_sentences, min_count=1)

w1 = "world"
w2 = "man"
w3 = w1

print(model.wv.similarity(w1,w2))
print("\n")
model.wv.most_similar(w3)

StarcoderdataJupyterScriptsDedupFiltered