# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + [markdown] id="pdqixpznsHrQ"
# # Topic Modelling
#
# Técnica de clasificación de documento no supervisada para agruparlos por temáticas similares.

# + id="kYQFVDhasHrZ" outputId="6d7a4160-0c0c-4b17-bb4c-c29538803ae6"
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.datasets import fetch_20newsgroups   #NLTK, gensim, 
from gensim.parsing.preprocessing import STOPWORDS

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer


from numpy.linalg import svd

dataset = fetch_20newsgroups(shuffle=True, random_state=1, remove=('headers', 'footers', 'quotes'))
docs    = dataset.data
len(docs)

# + id="WE12wmdssHrb" outputId="755abc66-b9a1-4b7a-bfef-f422717d4a3e"
print(dataset.DESCR)

# + id="yGf2FEm0sHrc" outputId="ce8845d9-1c26-4b10-bdad-bcbd785ba869"
dataset.target_names

# + id="l1ttWxmNsHrc"
df = pd.DataFrame({'documentos':docs})

# + id="iPFHgX8fsHrd" outputId="697a2353-2721-4c40-f604-027acfbab203"
STOPWORDS

# + id="MchMQo_AsHrd"
df['doc_limpio'] =  df['documentos'].str.replace("[^a-zA-Z#]", " ")
df['doc_limpio']  = df['doc_limpio'].apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]).lower() )
df['doc_limpio']  = df['doc_limpio'].apply(lambda x: ' '.join([w for w in x.split() if w not in STOPWORDS]))

# + id="Gal_7rhAsHrf" outputId="876f383e-b88a-4ae4-e1b7-00c9377a3fe4"
df

# + [markdown] id="ooMx85R-sHrg"
# ## Vectorización
#
# Para transformar los documentos en vectores hay por lo menos dos opciones. Una es el conteo de cada token en los documentos. Y otra es usando ``tf-idf`` (*Term Frequency - Inverse Document Frequency*)
#
# $$
# \operatorname{tf}(t,d) = 0.5 + 0.5\frac {f_{t,d}} {\max \{ f_{t',d} : t' \in D\}}
# $$
#
# y 
#
# $$
# \operatorname{idf}(t,D)  = \log \left(\frac {|D|} {1+ |\{d \in D : t \in D \} |} \right)
# $$
#
# entonces el _score_ por término es
#
# $$
# \operatorname{tf-idf}(t,d,D) = \operatorname{tf}(t,d) \times \operatorname{idf}(t,D)
# $$
#
# Vean el ejemplo en [Wikipedia](https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Example_of_tf%E2%80%93idf)

# + id="bg0vkpuSsHrh" outputId="b97799ec-ee6e-4b19-dbc2-d4e226845e47"
#vectorizador = CountVectorizer(stop_words='english', max_features=1000)
vectorizador = TfidfVectorizer(stop_words='english', max_features=1000, max_df=0.5, smooth_idf=True)
X            = vectorizador.fit_transform(df['doc_limpio'])
X

# + id="Zxlcp0ANsHrh" outputId="b60e5b8a-b286-4b3e-df9d-6fb189168ee1"
Xa  = X.toarray()
Xa[0]

# + id="KBSN0ZlQsHrh" outputId="3a1a332c-288d-46a4-c548-44d3dc33c250"
Xa.shape

# + id="t8CVH42LsHri"
u, s, vt = svd(Xa, full_matrices=False)

# + id="GSEc4KcisHri" outputId="6c0fbf3a-8dc6-42e8-d4cd-685fb8d2a53d"
u.shape

# + id="YMkgFDuTsHri" outputId="53ac9425-a7c6-4fa1-9800-6c294650a67c"
vt.shape

# + id="oLGSfdZ-sHrj" outputId="27201bbb-747e-4b70-ca02-fc763ae3f9f9"
s.shape

# + id="TNWf4XuKsHrj" outputId="4d8b0dc0-cac5-4252-effc-03f982580839"
reconstruye = lambda p,s,q,n : np.dot( 
                                      np.dot(p[:,:n], np.diag(s[:n])), 
                                      q[:n])

recomp = reconstruye(u,s,vt,1000)
recomp.shape


# + id="X4B2RBrgsHrj" outputId="f1b7ce29-2d31-4951-e380-aeb7f7fdbc19"
np.allclose(Xa, recomp)

# + id="_rV3EmNbsHrk" outputId="26efdb0d-0486-4ac3-da08-795b73f0d6b9"
sns.lineplot(x=range(1,len(s)+1), y=s)

# + id="2L-29PYmsHrk" outputId="f1f2fed2-ebb0-4b4a-dcaf-bb054f5320ae"
w = [s[i+1]/s[i] for i in range(len(s)-1)]
sns.lineplot(x = range(1,51), y=w[:50])

# + id="es6XD3s_sHrk" outputId="a1211775-ca47-420a-ab3a-9f70cf873eea"
R = reconstruye(u,s,vt, 10)
R.shape

# + id="29g1L1FCsHrl" outputId="a6b2f48a-0069-40dd-a878-1ac5a67efd90"
terms = vectorizador.get_feature_names()
len(terms)

# + id="bEwKG0y9sHrl" outputId="ad5516a2-93cc-4fee-ac8b-90dcc2b3c4ce"
print(df.iloc[2032]['doc_limpio'])

# + id="Mmwz8m60sHrl" outputId="a68a05b1-bb10-4e56-a827-10c75720412e"
vector = vectorizador.transform(['[deletions] [deletions] presented argument incredulity. however, seen, presented manner. usually presented form, "and *besides*, see... ...nor offered convincing explanation." moreover, unreasonable explanation phenomena. theism provide convincing explanation argument theisms favor. especially different theisms offer different explanations, different adherents purportedly theism different explanations... experience. experience, common reason lack evidence theisms favor. mileage vary. heck, ill snide once. its fairly easy attack arguments made. (i.e. strawmen.) sage advice indeed. sincerely, <NAME> <EMAIL>'])
varr   = vector.toarray()
varr.shape

# + id="tjcbzUtPsHrm" outputId="ff7dd0d0-9c88-466f-e44e-1d6874970935"
dh = np.dot(recomp, varr.T)
np.round(dh, decimals=3)

# + id="NQuAJWeNsHrm" outputId="e1085a7b-737b-4974-f62d-96bd0e46fbc9"
p = np.argsort(dh.reshape(1,-1))
p[0][-5:]

# + id="NdRjXgBysHrm" outputId="8d6ce3df-d2d9-4edd-fb24-7f2bc9daf60f"
print(df.iloc[10964]['doc_limpio'])

# + id="6SOEBt23sHrm" outputId="0108488e-fc6c-4812-a0e0-d665f0d1b7fb"
print(df.iloc[9528]['doc_limpio'])

# + id="914ljHJYsHrn" outputId="0cb662ed-44e8-4c7a-ac97-6d3c078c2a1a"
k = 10
vocab = vectorizador.vocabulary_
for i, componente in enumerate(vt):
    vocab_comp = zip(terms, componente)
    jerarquia_temas = sorted(vocab_comp, key=lambda x:x[1], reverse=True)[:k] 
    print(u"Tópico {}".format(i))
    for temas in jerarquia_temas:
        print("\t {} {}".format(temas[0], temas[1]))
    print("*"*50)

# + id="hpDWv5dTsHrn" outputId="b03b8741-0d3d-4750-a903-563fba2f6e3a"
len(vocab)

StarcoderdataJupyterScriptsDedupFiltered