code: # -*- coding: utf-8 -*-
  """
  Created on Mon Nov 11 13:41:14 2019
  
  @author: Emmett & Binyang
  """
  
  from pprint import pprint
  import pandas as pd
  import numpy as np
  import matplotlib.pyplot as plt
  import seaborn as sns
  from nltk.tokenize.punkt import PunktSentenceTokenizer, PunktTrainer
  
  ##Let’s first build a corpus to train our tokenizer on. We’ll use stuff available in NLTK:
  
  from nltk.corpus import gutenberg
  
  # print (dir(gutenberg))
  # print (gutenberg.fileids())
  
  text = ""
  for file_id in gutenberg.fileids():
      text += gutenberg.raw(file_id)
   
  print (len(text))
  
  ##a funtion that converts a list to a string
  def listToString(s):  
      
      # initialize an empty string 
      str1 = ""  
      
      # traverse in the string   
      for ele in s:  
          str1 += ele   
      
      # return string   
      return str1
  
  ##extract sentences from samples for following sentiment analysis
  sampNum = 1
  sent_df = pd.DataFrame()
  i = 0
  
  while (sampNum < 186):
      fileOpen = open("sample"+str(sampNum)+".txt","r")
      temp = fileOpen.readlines()
      temp = listToString(temp)
      
      trainer = PunktTrainer()
      trainer.INCLUDE_ALL_COLLOCS = True
      trainer.train(text)
      tokenizer = PunktSentenceTokenizer(trainer.get_params())
      
      ##Adding more abbreviations
      tokenizer._params.abbrev_types.add('dr')
   
      sent = tokenizer.tokenize(temp)
      
      for sent in sent:
          sent_df.loc[i, 'sent'] = sent
          sent_df.loc[i, 'sample'] = sampNum
          i += 1
      
      sampNum += 1
  
  ##NLTK’s built-in Vader Sentiment Analyzer will simply rank a piece of text as positive, negative or neutral 
  ##using a lexicon of positive and negative words.
  
  ##We can utilize this tool by first creating a Sentiment Intensity Analyzer (SIA) to categorize our headlines, 
  ##then we'll use the polarity_scores method to get the sentiment.
  
  ##We'll append each sentiment dictionary to a results list, which we'll transform into a dataframe:
  
  from nltk.sentiment.vader import SentimentIntensityAnalyzer as SIA
  
  sia = SIA()
  results = []
  
  for idx, row in sent_df.iterrows():
      line = row['sent']
      score = sia.polarity_scores(line)
      sent_df.loc[idx, 'neg'] = score.get('neg')
      sent_df.loc[idx, 'neu'] = score.get('neu')
      sent_df.loc[idx, 'pos'] = score.get('pos')
      sent_df.loc[idx, 'compound'] = score.get('compound')
  
  # pprint(results[:10], width=100)
  
  ##We will consider posts with a compound value greater than 0.2 as positive and less than -0.2 as negative. 
  ##There's some testing and experimentation that goes with choosing these ranges, and there is a trade-off to be 
  ##made here. If you choose a higher value, you might get more compact results (less false positives and false 
  ##negatives), but the size of the results will decrease significantly.
  
  sent_df['label'] = 0
  sent_df.loc[sent_df['compound'] > 0.3, 'label'] = 1
  sent_df.loc[sent_df['compound'] < -0.3, 'label'] = -1
  # sent_df.head()
  
  ##We have all the data we need to save, so let's do that:
  
  sent_df.to_csv('sentiment analysis.csv', mode='a', encoding='utf-8', index=False)
  
  ##We can now keep appending to this csv, but just make sure that if you reassign the headlines set, you could get 
  ##duplicates. Maybe add a more advanced saving function that reads and removes duplicates before saving.
  
  #Let's first take a peak at a few positive and negative headlines:
  
  print("Positive headlines:\n")
  pprint(list(sent_df[sent_df['label'] == 1].sent)[:5], width=200)
  
  print("\nNegative headlines:\n")
  pprint(list(sent_df[sent_df['label'] == -1].sent)[:5], width=200)
  
  ##Now let's check how many total positives and negatives we have in this dataset:
  
  print(sent_df.label.value_counts())
  print(sent_df.label.value_counts(normalize=True) * 100)
  
  ##The first line gives us raw value counts of the labels, whereas the second line provides percentages 
  ##with the normalize keyword.
  
  ##For fun, let's plot a bar chart:
  """
  fig, ax = plt.subplots(figsize=(8, 8))
  
  counts = sent_df.label.value_counts(normalize=True) * 100
  
  sns.barplot(x=counts.index, y=counts, ax=ax)
  
  ax.set_xticklabels(['Negative', 'Neutral', 'Positive'])
  ax.set_ylabel("Percentage")
  
  plt.show()
  """
  
  ##filter the sentences by number of words in it
  for idx, row in sent_df.iterrows():
      sentence = row['sent']
      sent_df.loc[idx, 'len_sent'] = len(sentence.split())
  
  ##split positive and other sentences
  pos = sent_df[sent_df['label'] == 1]
  neg = sent_df[sent_df['label'] != 1]
  
  import gensim
  from gensim.parsing.preprocessing import strip_non_alphanum
  from gensim.parsing.preprocessing import strip_punctuation
  from gensim.parsing.preprocessing import strip_multiple_whitespaces
  from gensim.parsing.preprocessing import stem_text
  
  corpus_full = []
  for idx, row in sent_df.iterrows():
      temp = row['sent']
      temp1 = strip_non_alphanum(str(temp))
      temp2 = strip_punctuation(temp1)
      temp3 = strip_multiple_whitespaces(temp2)
      final = stem_text(temp3)
      corpus_full.append(final)
  
  corpus_pos = []
  for idx, row in pos.iterrows():
      temp = row['sent']
      temp1 = strip_non_alphanum(str(temp))
      temp2 = strip_punctuation(temp1)
      temp3 = strip_multiple_whitespaces(temp2)
      final = stem_text(temp3)
      corpus_pos.append(final)
      
  corpus_neg = []
  for idx, row in neg.iterrows():
      temp = row['sent']
      temp1 = strip_non_alphanum(str(temp))
      temp2 = strip_punctuation(temp1)
      temp3 = strip_multiple_whitespaces(temp2)
      final = stem_text(temp3)
      corpus_neg.append(final)
  
  from nltk.corpus import stopwords
  stop_words = stopwords.words('english')
  
  stoplist = set('a about above after again against all am an and any are arent\
                 as also at be because been before being below between both but\
                 by cant cannot could couldnt did didnt do does doesnt doing dont\
                 down during each els few for from further had hadnt has have havent\
                 having he hed hes her here heres hers herself him himself his\
                 how hows i id ill im ive if in into is isnt it its itself lets\
                 me more most mustnt my myself no nor not of off on once only or\
                 other ought our ours ourselves out over own same shant she shes\
                 should shouldnt so some such than that thats the their theirs\
                 them themselves then there theres these they theyd theyll theyre\
                 theyve this those through to too under until up very was wasnt\
                 we wed were weve were werent what whats when whens which while\
                 who whos whom why whys with wont would wouldnt you youd youll\
                 youre youve your yours yourself yourselves ll ve s ar mayb ha re\
                 us thi isn a b c d e f g h i j k l m n o p q r s t u v w x y z\
                 hi will can get back go don wa let atc ok ani mi thei whenev make\
                 just take aw know sai good baltimor jetblu lol thank thanks like\
                 vari might less highest billion nice probabl lot fuck shit sure\
                 feel dure befor realli work veri chanc see awai onc onli dy aren\
                 100 someth thing even happen becaus wai everi much help want think\
                 fear flight plane fly mai time dai\
                 1 2 3 4 5 6 7 8 9 10'.split())
  
  print (len(stoplist))
  stoplist.update(stop_words)
  
  print(len(stop_words))
  print(len(stoplist))
  
  #standardize text -- makes all characters lowercase and removes common stop words
  text_full = [[word for word in document.lower().split() if word not in stoplist]
          for document in corpus_full]
  print(text_full)
  text_pos = [[word for word in document.lower().split() if word not in stoplist]
          for document in corpus_pos]
  text_neg = [[word for word in document.lower().split() if word not in stoplist]
          for document in corpus_neg]
  
  #count number of times that word appears in corpus
  #pair frequency with respective word in new array
  from collections import defaultdict
  
  frequency = defaultdict(int)
  for text in text_full:
      for token in text:
          frequency[token] += 1
  
  corpus_removeOne_full = [[token for token in text if frequency[token]>1] for text in text_full]
  
  frequency = defaultdict(int)
  for text in text_pos:
      for token in text:
          frequency[token] += 1
          
  corpus_removeOne_pos = [[token for token in text if frequency[token]>1] for text in text_pos]
  
  frequency = defaultdict(int)
  for text in text_neg:
      for token in text:
          frequency[token] += 1
          
  corpus_removeOne_neg = [[token for token in text if frequency[token]>1] for text in text_neg]
  
  
  from gensim import corpora
  #add corpora to dictionary
  dictionary_full = corpora.Dictionary(corpus_removeOne_full)
  dictionary_pos = corpora.Dictionary(corpus_removeOne_pos)
  dictionary_neg = corpora.Dictionary(corpus_removeOne_neg)
  #save dictionary for future reference
  dictionary_full.save('redditTest_full.dict')
  dictionary_pos.save('redditTest_pos.dict') #location of document in computer
  dictionary_neg.save('redditTest_neg.dict')
  #dict = gensim.corpora.Dictionary.load('redditTest.dict')
  
  #assign numeric id to each token in dictionary
  dictID_full = dictionary_full.token2id
  dictID_pos = dictionary_pos.token2id
  dictID_neg = dictionary_neg.token2id
  
  #remove empty sentences
  for text in corpus_removeOne_full:
      if len(text) == 0:
          corpus_removeOne_full.remove(text)
  
  for text in corpus_removeOne_pos:
      if len(text) == 0:
          corpus_removeOne_pos.remove(text)
          
  for text in corpus_removeOne_neg:
      if len(text) == 0:
          corpus_removeOne_neg.remove(text)
  
  
  #converts each word into vector following same process as example
  #Bag of Word Corpus of Full Sentiment
  bow_corpus_full = [dictionary_full.doc2bow(text) for text in corpus_removeOne_full]
  corpora.MmCorpus.serialize('redditTest_full.mm', bow_corpus_full)
  corp_full = gensim.corpora.MmCorpus('redditTest_full.mm')
  
  from gensim import models
  tfidf_pos = models.TfidfModel(bow_corpus_full)
  corpus_tfidf_full = tfidf_pos[bow_corpus_full]
  
  #Bag of Word Corpus of Positive Sentiment
  bow_corpus_pos = [dictionary_pos.doc2bow(text) for text in corpus_removeOne_pos]
  corpora.MmCorpus.serialize('redditTest_pos.mm', bow_corpus_pos)
  corp_pos = gensim.corpora.MmCorpus('redditTest_pos.mm')
  
  from gensim import models
  tfidf_pos = models.TfidfModel(bow_corpus_pos)
  corpus_tfidf_pos = tfidf_pos[bow_corpus_pos]
  
  #Bag of Word Corpus of Negative Sentiment
  bow_corpus_neg = [dictionary_neg.doc2bow(text) for text in corpus_removeOne_neg]
  corpora.MmCorpus.serialize('redditTest_neg.mm', bow_corpus_neg)
  corp_neg = gensim.corpora.MmCorpus('redditTest_neg.mm')
  
  from gensim import models
  tfidf_neg = models.TfidfModel(bow_corpus_neg)
  corpus_tfidf_neg = tfidf_neg[bow_corpus_neg]
  
  
  #LDA Mallet for full corpus
  mallet_path = '/Users/emmet/.spyder-py3-dev/REU_Project/mallet-2.0.8/bin/mallet'
  lda_full = gensim.models.wrappers.LdaMallet(mallet_path, corpus=bow_corpus_full, num_topics=9, id2word=dictionary_full, workers=1, alpha=110, random_seed=109, iterations=50)
  corpus_LDA_full = lda_full[bow_corpus_full]
  lda_full.print_topics(9)
  
  #LDA Mallet for positive corpus
  mallet_path = '/Users/emmet/.spyder-py3-dev/REU_Project/mallet-2.0.8/bin/mallet'
  lda_pos = gensim.models.wrappers.LdaMallet(mallet_path, corpus=bow_corpus_pos, num_topics=9, id2word=dictionary_pos, workers=1, alpha=110, random_seed=109, iterations=50)
  corpus_LDA_pos = lda_pos[bow_corpus_pos]
  lda_pos.print_topics(9)
  
  #LDA Mallet for negative corpus
  mallet_path = '/Users/emmet/.spyder-py3-dev/REU_Project/mallet-2.0.8/bin/mallet'
  lda_neg = gensim.models.wrappers.LdaMallet(mallet_path, corpus=bow_corpus_neg, num_topics=9, id2word=dictionary_neg, workers=1, alpha=110, random_seed=109, iterations=50)
  corpus_LDA_neg = lda_neg[bow_corpus_neg]
  lda_neg.print_topics(9)
  
  
  import pandas as pd
  import matplotlib.pyplot as plt
  import matplotlib.colors as mcolors
  from sklearn.manifold import TSNE
  
  colors = np.array([color for name, color in mcolors.TABLEAU_COLORS.items()])
  
  #t-SNE plot for full corpus
  n_topics = 9
  topic_weights_full = []
  for row_list in lda_full[bow_corpus_full]:
      tmp = np.zeros(n_topics)
      for i, w in row_list:
          tmp[i] = w
      topic_weights_full.append(tmp)
      
  arr_full = pd.DataFrame(topic_weights_full).fillna(9).values
  topic_num_full = np.argmax(arr_full, axis=1)
  tsne_model_full = TSNE(n_components=3, random_state=None, method='barnes_hut', 
                    angle=0.5, init='pca')
  tsne_lda_full = tsne_model_full.fit_transform(arr_full)
  
  sub = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")
  plt.xlabel('t-SNE1'.translate(sub))
  plt.ylabel('t-SNE2'.translate(sub))
  plt.title('t-SNE Plot of Topics within Positive Sentiment Corpus')
  tsne_full = plt.scatter(x=tsne_lda_full[:,0], y=tsne_lda_full[:,1])
  plt.show(tsne_full)
  
  """
  #t-SNE plot for positive corpus
  n_topics = 9
  topic_weights_pos = []
  for row_list in lda_pos[bow_corpus_pos]:
      tmp = np.zeros(n_topics)
      for i, w in row_list:
          tmp[i] = w
      topic_weights_pos.append(tmp)
      
  arr_pos = pd.DataFrame(topic_weights_pos).fillna(0).values
  topic_num_pos = np.argmax(arr_pos, axis=1)
  tsne_model_pos = TSNE(n_components=3, random_state=None, method='barnes_hut', 
                    angle=0.5, init='pca')
  tsne_lda_pos = tsne_model_pos.fit_transform(arr_pos)
  
  sub = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")
  plt.xlabel('t-SNE1'.translate(sub))
  plt.ylabel('t-SNE2'.translate(sub))
  plt.title('t-SNE Plot of Topics within Positive Sentiment Corpus')
  tsne_pos = plt.scatter(x=tsne_lda_pos[:,0], y=tsne_lda_pos[:,1])
  #plt.show(tsne_pos)
  
  
  #t-SNE plot for negative corpus
  n_topics = 9
  topic_weights_neg = []
  for row_list in lda_neg[bow_corpus_neg]:
      tmp = np.zeros(n_topics)
      for i, w in row_list:
          tmp[i] = w
      topic_weights_neg.append(tmp)
      
  arr_neg = pd.DataFrame(topic_weights_neg).fillna(0).values
  topic_num_neg = np.argmax(arr_neg, axis=1)
  tsne_model_neg = TSNE(n_components=3, random_state=None, method='barnes_hut', 
                    angle=0.5, init='pca')
  tsne_lda_neg = tsne_model_neg.fit_transform(arr_neg)
  
  sub = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉")
  plt.xlabel('t-SNE1'.translate(sub))
  plt.ylabel('t-SNE2'.translate(sub))
  plt.title('t-SNE Plot of Topics within Negative Sentiment Corpus')
  tsne_neg = plt.scatter(tsne_lda_neg[:,0], tsne_lda_neg[:,1])
  #plt.show(tsne_neg)
  """
  
  from collections import Counter
  #Word Count & Keyword for Full Corpus
  topics_full = lda_full.show_topics(formatted=False)
  flatten_full = [w for w_list in bow_corpus_full for w in w_list]
  counter_full = Counter(flatten_full)
  
  topic_weight_full = []
  for i, topic in topics_full:
      for word, weight in topic:
          topic_weight_full.append([word, i , weight, counter_full[word]])
  
  data_frame_full = pd.DataFrame(topic_weight_full, columns=['word', 'topic_id', 'importance', 'word_count'])        
  
  fig, axes = plt.subplots(3, 3, figsize=(10,6), sharey=True, dpi=160)
  
  for i, ax in enumerate(axes.flatten()):
      ax.bar(x='word', height="word_count", data=data_frame_full.loc[data_frame_full.topic_id==i, :], color=colors[i], width=0.5, alpha=0.3, label='Word Count')
      ax_twin = ax.twinx()
      ax_twin.bar(x='word', height="importance", data=data_frame_full.loc[data_frame_full.topic_id==i, :], color=colors[i], width=0.2, label='Weights')
      ax.set_ylabel('Word Count', color=colors[i])
      ax_twin.set_ylim(0, 0.5); ax.set_ylim(0, 100)
      ax.set_title('Topic: ' + str(i+1), color=colors[i], fontsize=8)
      ax.tick_params(axis='y', left=False)
      ax.set_xticklabels(data_frame_full.loc[data_frame_full.topic_id==i, 'word'], rotation=90, horizontalalignment= 'center')
      ax.legend(loc='upper left'); ax_twin.legend(loc='upper right')
  
  fig.tight_layout(w_pad=2)    
  plt.show()
  
  """
  #Word Count & Keyword for Positive Corpus
  topics_pos = lda_pos.show_topics(formatted=False)
  flatten_pos = [w for w_list in bow_corpus_pos for w in w_list]
  counter_pos = Counter(flatten_pos)
  
  topic_weight_pos = []
  for i, topic in topics_pos:
      for word, weight in topic:
          topic_weight_pos.append([word, i , weight, counter_pos[word]])
  
  data_frame_pos = pd.DataFrame(topic_weight_pos, columns=['word', 'topic_id', 'importance', 'word_count'])        
  
  fig, axes = plt.subplots(3, 3, figsize=(10,6), sharey=True, dpi=160)
  
  for i, ax in enumerate(axes.flatten()):
      ax.bar(x='word', height="word_count", data=data_frame_pos.loc[data_frame_pos.topic_id==i, :], color=colors[i], width=0.5, alpha=0.3, label='Word Count')
      ax_twin = ax.twinx()
      ax_twin.bar(x='word', height="importance", data=data_frame_pos.loc[data_frame_pos.topic_id==i, :], color=colors[i], width=0.2, label='Weights')
      ax.set_ylabel('Word Count', color=colors[i])
      ax_twin.set_ylim(0, 0.5); ax.set_ylim(0, 100)
      ax.set_title('Topic: ' + str(i+1), color=colors[i], fontsize=8)
      ax.tick_params(axis='y', left=False)
      ax.set_xticklabels(data_frame_pos.loc[data_frame_pos.topic_id==i, 'word'], rotation=90, horizontalalignment= 'center')
      ax.legend(loc='upper left'); ax_twin.legend(loc='upper right')
  
  fig.tight_layout(w_pad=2)    
  plt.show()
  
  #Word Count & Keyword for Negative Corpus
  topics_neg = lda_neg.show_topics(formatted=False)
  flatten_neg = [w for w_list in bow_corpus_neg for w in w_list]
  counter_neg = Counter(flatten_neg)
  
  topic_weight_neg = []
  for i, topic in topics_neg:
      for word, weight in topic:
          topic_weight_neg.append([word, i , weight, counter_neg[word]])
  
  data_frame_neg = pd.DataFrame(topic_weight_neg, columns=['word', 'topic_id', 'importance', 'word_count'])        
  
  fig, axes = plt.subplots(3, 3, figsize=(10,6), sharey=True, dpi=160)
  
  for i, ax in enumerate(axes.flatten()):
      ax.bar(x='word', height="word_count", data=data_frame_neg.loc[data_frame_neg.topic_id==i, :], color=colors[i], width=0.5, alpha=0.3, label='Word Count')
      ax_twin = ax.twinx()
      ax_twin.bar(x='word', height="importance", data=data_frame_neg.loc[data_frame_neg.topic_id==i, :], color=colors[i], width=0.2, label='Weights')
      ax.set_ylabel('Word Count', color=colors[i])
      ax_twin.set_ylim(0, 0.5); ax.set_ylim(0, 100)
      ax.set_title('Topic: ' + str(i+1), color=colors[i], fontsize=8)
      ax.tick_params(axis='y', left=False)
      ax.set_xticklabels(data_frame_neg.loc[data_frame_neg.topic_id==i, 'word'], rotation=90, horizontalalignment= 'center')
      ax.legend(loc='upper left'); ax_twin.legend(loc='upper right')
  
  fig.tight_layout(w_pad=2)    
  plt.show()
  """
  
  from wordcloud import WordCloud
  import matplotlib.colors as mcolors
  #Word Cloud Display for Full Corpus
  cloud = WordCloud(stopwords=stoplist, background_color='white', width=2500, height=1800, max_words=7, colormap='tab10', color_func=lambda *args, **kwargs: colors[i], prefer_horizontal=1.0)
  
  topics_full = lda_full.show_topics(formatted=False)
  
  fig, axes = plt.subplots(3, 3, figsize=(10, 6))
  
  for i, ax in enumerate(axes.flatten()):
      fig.add_subplot(ax)
      topic_words_full = dict(topics_full[i][1])
      cloud.generate_from_frequencies(topic_words_full, max_font_size=300)
      plt.gca().imshow(cloud)
      plt.gca().set_title('Topic ' + str(i+1), fontdict=dict(size=10))
      plt.gca().axis('off')
  
  plt.axis('off')
  plt.tight_layout()
  plt.show()
  
  """
  #Word Cloud Display for Positive Corpus
  cloud = WordCloud(stopwords=stoplist, background_color='white', width=2500, height=1800, max_words=7, colormap='tab10', color_func=lambda *args, **kwargs: colors[i], prefer_horizontal=1.0)
  
  topics_pos = lda_pos.show_topics(formatted=False)
  
  fig, axes = plt.subplots(3, 3, figsize=(10, 6))
  
  for i, ax in enumerate(axes.flatten()):
      fig.add_subplot(ax)
      topic_words_pos = dict(topics_pos[i][1])
      cloud.generate_from_frequencies(topic_words_pos, max_font_size=300)
      plt.gca().imshow(cloud)
      plt.gca().set_title('Topic ' + str(i+1), fontdict=dict(size=10))
      plt.gca().axis('off')
  
  plt.axis('off')
  plt.tight_layout()
  plt.show()
  
  #Word Cloud Display for Negative Corpus
  cloud = WordCloud(stopwords=stoplist, background_color='white', width=2500, height=1800, max_words=7, colormap='tab10', color_func=lambda *args, **kwargs: colors[i], prefer_horizontal=1.0)
  
  topics_neg = lda_neg.show_topics(formatted=False)
  
  fig, axes = plt.subplots(3, 3, figsize=(10, 6))
  
  for i, ax in enumerate(axes.flatten()):
      fig.add_subplot(ax)
      topic_words_neg = dict(topics_neg[i][1])
      cloud.generate_from_frequencies(topic_words_neg, max_font_size=300)
      plt.gca().imshow(cloud)
      plt.gca().set_title('Topic ' + str(i+1), fontdict=dict(size=10))
      plt.gca().axis('off')
  
  plt.axis('off')
  plt.tight_layout()
  plt.show()
  """
  
  import pyLDAvis.gensim
  import pyLDAvis
  import gensim   
  
  #LDA Mallet pyLDAvis for Full Corpus
  mallet2lda_full = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(lda_full)
  visualizeLDA_full = pyLDAvis.gensim.prepare(mallet2lda_full, bow_corpus_full, dictionary_full)
  pyLDAvis.show()
  
  """
  #LDA Mallet pyLDAvis for Postiive Corpus
  mallet2lda_pos = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(lda_pos)
  visualizeLDA_pos = pyLDAvis.gensim.prepare(mallet2lda_pos, bow_corpus_pos, dictionary_pos)
  pyLDAvis.show(visualizeLDA_pos)
  
  #LDA Mallet pyLDAvis for Negative Corpus
  mallet2lda_neg = gensim.models.wrappers.ldamallet.malletmodel2ldamodel(lda_neg)
  visualizeLDA_neg = pyLDAvis.gensim.prepare(mallet2lda_neg, bow_corpus_neg, dictionary_neg)
  pyLDAvis.show(visualizeLDA_neg)
  """ # -*- coding: utf-8 -*-
  """
  Created on Thu Nov  5 22:53:44 2020
  
  @author: Emmett
  """
  import tensorflow as tf
  import os
  from tensorflow.python.keras.layers import Layer
  from tensorflow.python.keras import backend as K
  
  
  class AttentionLayer(Layer):
      """
      This class implements Bahdanau attention (https://arxiv.org/pdf/1409.0473.pdf).
      There are three sets of weights introduced W_a, U_a, and V_a
       """
  
      def __init__(self, **kwargs):
          super(AttentionLayer, self).__init__(**kwargs)
  
      def build(self, input_shape):
          assert isinstance(input_shape, list)
          # Create a trainable weight variable for this layer.
  
          self.W_a = self.add_weight(name='W_a',
                                     shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),
                                     initializer='uniform',
                                     trainable=True)
          self.U_a = self.add_weight(name='U_a',
                                     shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),
                                     initializer='uniform',
                                     trainable=True)
          self.V_a = self.add_weight(name='V_a',
                                     shape=tf.TensorShape((input_shape[0][2], 1)),
                                     initializer='uniform',
                                     trainable=True)
  
          super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end
  
      def call(self, inputs, verbose=False):
          """
          inputs: [encoder_output_sequence, decoder_output_sequence]
          """
          assert type(inputs) == list
          encoder_out_seq, decoder_out_seq = inputs
          if verbose:
              print('encoder_out_seq>', encoder_out_seq.shape)
              print('decoder_out_seq>', decoder_out_seq.shape)
  
          def energy_step(inputs, states):
              """ Step function for computing energy for a single decoder state
              inputs: (batchsize * 1 * de_in_dim)
              states: (batchsize * 1 * de_latent_dim)
              """
  
              assert_msg = "States must be an iterable. Got {} of type {}".format(states, type(states))
              assert isinstance(states, list) or isinstance(states, tuple), assert_msg
  
              """ Some parameters required for shaping tensors"""
              en_seq_len, en_hidden = encoder_out_seq.shape[1], encoder_out_seq.shape[2]
              de_hidden = inputs.shape[-1]
  
              """ Computing S.Wa where S=[s0, s1, ..., si]"""
              # <= batch size * en_seq_len * latent_dim
              W_a_dot_s = K.dot(encoder_out_seq, self.W_a)
  
              """ Computing hj.Ua """
              U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim
              if verbose:
                  print('Ua.h>', U_a_dot_h.shape)
  
              """ tanh(S.Wa + hj.Ua) """
              # <= batch_size*en_seq_len, latent_dim
              Ws_plus_Uh = K.tanh(W_a_dot_s + U_a_dot_h)
              if verbose:
                  print('Ws+Uh>', Ws_plus_Uh.shape)
  
              """ softmax(va.tanh(S.Wa + hj.Ua)) """
              # <= batch_size, en_seq_len
              e_i = K.squeeze(K.dot(Ws_plus_Uh, self.V_a), axis=-1)
              # <= batch_size, en_seq_len
              e_i = K.softmax(e_i)
  
              if verbose:
                  print('ei>', e_i.shape)
  
              return e_i, [e_i]
  
          def context_step(inputs, states):
              """ Step function for computing ci using ei """
  
              assert_msg = "States must be an iterable. Got {} of type {}".format(states, type(states))
              assert isinstance(states, list) or isinstance(states, tuple), assert_msg
  
              # <= batch_size, hidden_size
              c_i = K.sum(encoder_out_seq * K.expand_dims(inputs, -1), axis=1)
              if verbose:
                  print('ci>', c_i.shape)
              return c_i, [c_i]
  
          fake_state_c = K.sum(encoder_out_seq, axis=1)
          fake_state_e = K.sum(encoder_out_seq, axis=2)  # <= (batch_size, enc_seq_len, latent_dim
  
          """ Computing energy outputs """
          # e_outputs => (batch_size, de_seq_len, en_seq_len)
          last_out, e_outputs, _ = K.rnn(
              energy_step, decoder_out_seq, [fake_state_e],
          )
  
          """ Computing context vectors """
          last_out, c_outputs, _ = K.rnn(
              context_step, e_outputs, [fake_state_c],
          )
  
          return c_outputs, e_outputs
  
      def compute_output_shape(self, input_shape):
          """ Outputs produced by the layer """
          return [
              tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),
              tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))
          ]
apis: numpy.zeros matplotlib.pyplot.axis pandas.DataFrame matplotlib.pyplot.tight_layout matplotlib.colors.TABLEAU_COLORS.items numpy.argmax matplotlib.pyplot.subplots matplotlib.pyplot.title sklearn.manifold.TSNE matplotlib.pyplot.gca matplotlib.pyplot.show matplotlib.pyplot.scatter
apis: tensorflow.python.keras.backend.softmax tensorflow.python.keras.backend.expand_dims tensorflow.python.keras.backend.rnn tensorflow.python.keras.backend.tanh tensorflow.TensorShape tensorflow.python.keras.backend.dot tensorflow.python.keras.backend.sum