# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     name: python3
# ---

# + id="l78yxv3B6ul2" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606014437069, "user_tz": -480, "elapsed": 1166, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}} outputId="a13137b5-ecea-44a2-fe0c-7ab2257ec331"
from google.colab import drive
drive.mount('/content/gdrive')
import os
os.chdir('/content/gdrive/My Drive/finch/sklearn/topic_modelling/imdb')

# + id="yqACtJjhHQJV" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606014440003, "user_tz": -480, "elapsed": 4030, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}} outputId="657c3e91-9e6b-45a0-e9a5-72f43ac72139"
# !pip install pyldavis

# + id="smbQz4Aj83H2" executionInfo={"status": "ok", "timestamp": 1606014441978, "user_tz": -480, "elapsed": 5831, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}}
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import TfidfVectorizer

import tensorflow as tf


# + id="L79WJovn-lPx" executionInfo={"status": "ok", "timestamp": 1606014442039, "user_tz": -480, "elapsed": 3802, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}}
def get_idx2word(_index_from=3):
  word2idx = tf.keras.datasets.imdb.get_word_index()
  word2idx = {k:(v+_index_from) for k,v in word2idx.items()}
  word2idx["<pad>"] = 0
  word2idx["<start>"] = 1
  word2idx["<unk>"] = 2
  idx2word = {idx: w for w, idx in word2idx.items()}
  return idx2word


# + id="IvXGOaP38TBj" colab={"base_uri": "https://localhost:8080/"} executionInfo={"status": "ok", "timestamp": 1606016630725, "user_tz": -480, "elapsed": 2188574, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}} outputId="1af1b15a-1190-424b-d36a-56e80180b936"
N_TOPICS = 10
MAX_TERMS = 10

(X_train, y_train), (X_test, y_test) = tf.keras.datasets.imdb.load_data()
idx2word = get_idx2word()
documents = [' '.join([idx2word[idx] for idx in x_train[1:]]) for x_train in X_train] + \
            [' '.join([idx2word[idx] for idx in x_test[1:]]) for x_test in X_test]

tfidf = TfidfVectorizer(stop_words = 'english',
                        max_df = .1,
                        max_features = 5000)
tfidf_repr = tfidf.fit_transform(documents)

lda = LatentDirichletAllocation(n_components = N_TOPICS,
                                learning_method = 'batch',
                                max_iter = 100,
                                evaluate_every = 1,
                                verbose = 1)
lda.fit(tfidf_repr)

for topic_idx, term_vals in enumerate(lda.components_):
  message = "Topic #{}: ".format(topic_idx)
  message += " ".join([tfidf.get_feature_names()[i]
                       for i in term_vals.argsort()[:-MAX_TERMS-1:-1]])
  print(message)

# + id="LLa6ogEEHcjk" executionInfo={"status": "ok", "timestamp": 1606016631178, "user_tz": -480, "elapsed": 2186999, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}}
import pyLDAvis
import pyLDAvis.sklearn

# + id="VJaNY7YPHe_N" executionInfo={"status": "ok", "timestamp": 1606016646139, "user_tz": -480, "elapsed": 2200511, "user": {"displayName": "\u5982\u5b50", "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gi3ItGjzEGzUOlXTUHjOgeuVA5TICdNcY-Q1TGicA=s64", "userId": "01997730851420384589"}}
pyLDAvis.save_html(pyLDAvis.sklearn.prepare(lda, tfidf_repr, tfidf), 'lda.html')

StarcoderdataJupyterScriptsDedupFiltered