text: # -*- coding: utf-8 -*-
  """main.ipynb
  
  Automatically generated by Colaboratory.
  
  Original file is located at
      https://colab.research.google.com/drive/1l8FPPWGor6rFXkLSiQD_lXi3ZcJSct-I
  
  # Biomedical Text Analytics
  
  ## Importing the Libraries
  """
  
  """ Created on 4 April 2019,
  Modified on 26 April 2019
  
  @Author: Anoushkrit Goel 
  """
  !pip install syspath
  import sys
  !pip install re2
  import re
  import numpy as np
  
  !pip install pprint 
  from pprint import pprint
  import pandas as pd
  # Bio
  !pip install biopython
  !pip install bio
  
  import bio
  from Bio import Entrez, Medline
  
  
  
  # Gensim
  !pip install gensim 
  import gensim
  import gensim.corpora as corpora 
  from gensim.utils import simple_preprocess
  
  from gensim.models import CoherenceModel # Coherence Model may not be used
  
  # spacy for Lemmatization
  !pip install spacy
  !python -m spacy download en
  import spacy
  
  #Plotting Tools
  !pip install pyLDAvis
  import pyLDAvis 
  import pyLDAvis.gensim #crucial step 
  import matplotlib.pyplot as plt
  # %matplotlib inline
  
  !pip install seaborn
  import seaborn as sns
  import matplotlib.colors as mcolors
  from matplotlib.ticker import FuncFormatter
  
  # Network Visualisation 
  !pip install networkx
  import networkx as nx
  
  #Quality Analysis and Testing Purposes
  !pip install logging 
  import logging
  logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.ERROR)
  
  !pip install pytest-warnings
  import warnings
  warnings.filterwarnings("ignore", category=DeprecationWarning)
  
  # File Conversion 
  !pip install python-csv
  !pip install jsonlib
  
  import csv
  import json
  !pip install collections-extended
  from collections import Counter
  from collections import defaultdict
  
  # Email for PubMed Verification 
  Entrez.email = "anoushkritgoel@gmail.com"
  
  """### Search Database
  Function to search the respective database.
  """
  
  def searchpubmed(return_max= None, database=None, searchqueryinput=None):
    print('Search' + database)
    if searchqueryinput:
      searchquery = searchqueryinput
    handle = Entrez.esearch(db=database, term=searchquery, retmax=return_max)  # retmax is 20 by esearch default
    queryresponse = Entrez.read(handle)
    handle.close()
    print("\n" + queryresponse['Count'] + " results available for \'" + str(searchquery) + "\', returning " + str(
          return_max))
    queryids = (searchquery, queryresponse['IdList'])
    return queryids
  
  """###Fetch Record"""
  
  def fetchrecord(inputids, numberofrecs, queryinput):
      print('Downloading Pubmed Records...')
      handle = Entrez.efetch(database, id=str(inputids), rettype="medline", retmode="text")
      records = Medline.parse(handle)
      records = list(records)
      recordsdf = pd.DataFrame(records)
      print('\nFeatures available: {}'.format(recordsdf.columns.values.tolist()))
      recordsdf['Searched'] = str(queryinput)
      reckeys = ['PMID', 'TI', 'AB', 'DP', 'PHST', 'Searched']
      print("\nReturning {} features : {} ".format((len(reckeys) - 1), reckeys[0:(len(reckeys) - 1)]))
      recordsselectdf = recordsdf[reckeys]
      
      recordsdf.to_json('full_records' + str(numberofrecs) + '.json')
      recordsselectdf.to_json('selected_records' + str(numberofrecs) + '.json')
      return recordsselectdf
  
  numberofrecs= input("Number of records?")
  queryinput= input("Which disease you want to visualize?")
  database = input("Which database to refer?")
  inputids = searchpubmed(return_max=numberofrecs, database=database, searchqueryinput=queryinput)
  fetchrecord(inputids, numberofrecs, queryinput)
  
  """### NLTK Stop Words"""
  
  # NLTK Stop Words
  import nltk
  nltk.download('stopwords')
  from nltk.corpus import stopwords
  stop_words = stopwords.words('english')
  ## This is respective of the corpus and the domain from which the individual is fetching data
  stop_words.extend(['from'])
  
  """### CSV to JSON"""
  
  def csv_json(csv=None,json='pubmed.json' ):
      csvfile = open(csv, 'r')
      jsonfile = open(json, 'w')
      ### Specify the field names based on the Headers required
      ### For Medical Literature headers lije
      fieldnames = ("PMID", "TI", "AB", "DP", "PHST")
      reader = csv.DictReader( csvfile, fieldnames)
      out = json.dumps( [ row for row in reader ] )
      jsonfile.write(out)
      return json
  
  """##Importing Dataset
  The dataset needs to be imported in JSON format and not in .CSV as the JSON provides us more flexibility with data by asssigning asymmetric value allocation in each and every cell unit.
  """
  
  # Import Dataset
  #### Specify the CSV name and JSON made
  df = pd.read_json('selected_records' + str(numberofrecs) + '.json')
  # df = df.loc[df.isin(['PMID', 'TI', 'AB', 'DP']) , :]
  print(df.shape)  #> (2361, 3)
  df.head()
  
  """### Tokenize Sentences and Cleaning"""
  
  def sent_to_words(sentences):
      for sent in sentences:
          sent = re.sub('\S*@\S*\s?', '', sent)  # remove emails
          sent = re.sub('\s+', ' ', sent)  # remove newline chars
          sent = re.sub("\'", "", sent)  # remove single quotes
          sent = gensim.utils.simple_preprocess(str(sent), deacc=True) 
          yield(sent)
  
  # Convert to list
  df['content'] = df['AB'] + df['TI']
  data = df.content.values.tolist()
  data_words = list(sent_to_words(df.content.apply(str)))
  pprint(data[:1])
  
  # for x,row in df.iterrows():
    # content = df['AB']+df['TI']
    #content = str(content)
    # print(content)
    #data = df.content.values.tolist()
    #data_words = list(sent_to_words(content))
    #print(data_words)
  
  def sent_to_words(sentences):
      for sentence in sentences:
          yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations
  
  data_words = list(sent_to_words(data))
  
  print(data_words[:1])
  
  """###BI-GRAM and TRI-GRAM Models"""
  
  # Build the bigram and trigram models
  bigram = gensim.models.Phrases(data_words, min_count=5, threshold=100) # higher threshold fewer phrases.
  trigram = gensim.models.Phrases(bigram[data_words], threshold=100)  
  bigram_mod = gensim.models.phrases.Phraser(bigram)
  trigram_mod = gensim.models.phrases.Phraser(trigram)
  
  # !python3 -m spacy download en  # run in terminal once
  def process_words(texts, stop_words=stop_words, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV']):
      """Remove Stopwords, Form Bigrams, Trigrams and Lemmatization"""
      texts = [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts]
      texts = [bigram_mod[doc] for doc in texts]
      texts = [trigram_mod[bigram_mod[doc]] for doc in texts]
      texts_out = []
      nlp = spacy.load('en', disable=['parser', 'ner'])
      for sent in texts:
          doc = nlp(" ".join(sent)) 
          texts_out.append([token.lemma_ for token in doc if token.pos_ in allowed_postags])
      # remove stopwords once more after lemmatization
      texts_out = [[word for word in simple_preprocess(str(doc)) if word not in stop_words] for doc in texts_out]    
      return texts_out
  
  data_ready = process_words(data_words)  # processed Text Data!
  
  """###Build the Topic Model"""
  
  # Create Dictionary
  id2word = corpora.Dictionary(data_ready)
  
  # Create Corpus: Term Document Frequency
  corpus = [id2word.doc2bow(text) for text in data_ready]
  
  # Build LDA model
  lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                             id2word=id2word,
                                             num_topics=4, 
                                             random_state=100,
                                             update_every=1,
                                             chunksize=10,
                                             passes=10,
                                             alpha='symmetric',
                                             iterations=100,
                                             per_word_topics=True)
  
  pprint(lda_model.print_topics())
  
  """###Dominant Topic Extraction
  Determining the Dominant Topic and further calculating the percentage contribution in each document.
  """
  
  def format_topics_sentences(ldamodel=None, corpus=corpus, texts=data):
      # Init output
      sent_topics_df = pd.DataFrame()
  
      # Get main topic in each document
      for i, row_list in enumerate(ldamodel[corpus]):
          row = row_list[0] if ldamodel.per_word_topics else row_list            
          # print(row)
          row = sorted(row, key=lambda x: (x[1]), reverse=True)
          # Get the Dominant topic, Perc Contribution and Keywords for each document
          for j, (topic_num, prop_topic) in enumerate(row):
              if j == 0:  # => dominant topic
                  wp = ldamodel.show_topic(topic_num)
                  topic_keywords = ", ".join([word for word, prop in wp])
                  sent_topics_df = sent_topics_df.append(pd.Series([int(topic_num), round(prop_topic,4), topic_keywords]), ignore_index=True)
              else:
                  break
      sent_topics_df.columns = ['Dominant_Topic', 'Perc_Contribution', 'Topic_Keywords']
  
      # Add original text to the end of the output
      contents = pd.Series(texts)
      sent_topics_df = pd.concat([sent_topics_df, contents], axis=1)
      return(sent_topics_df)
  
  
  df_topic_sents_keywords = format_topics_sentences(ldamodel=lda_model, corpus=corpus, texts=data_ready)
  
  # Format
  df_dominant_topic = df_topic_sents_keywords.reset_index()
  df_dominant_topic.columns = ['Document_No', 'Dominant_Topic', 'Topic_Perc_Contrib', 'Keywords', 'Text']
  df_dominant_topic.head(10)
  
  """###How can I tell about a topic in one sentence?"""
  
  # Display setting to show more characters in column
  pd.options.display.max_colwidth = 100
  
  sent_topics_sorteddf_mallet = pd.DataFrame()
  sent_topics_outdf_grpd = df_topic_sents_keywords.groupby('Dominant_Topic')
  
  for i, grp in sent_topics_outdf_grpd:
      sent_topics_sorteddf_mallet = pd.concat([sent_topics_sorteddf_mallet, 
                                               grp.sort_values(['Perc_Contribution'], ascending=False).head(1)], 
                                              axis=0)
  
  # Reset Index    
  sent_topics_sorteddf_mallet.reset_index(drop=True, inplace=True)
  
  # Format
  sent_topics_sorteddf_mallet.columns = ['Topic_Num', "Topic_Perc_Contrib", "Keywords", "Representative Text"]
  
  # Show
  sent_topics_sorteddf_mallet.head(10)
  
  """###Frequency Distribution of Word Counts in Documents"""
  
  doc_lens = [len(d) for d in df_dominant_topic.Text]
  
  # Plot
  plt.figure(figsize=(16,7), dpi=160)
  plt.hist(doc_lens, bins = 1000, color='navy')
  plt.text(750, 100, "Mean   : " + str(round(np.mean(doc_lens))))
  plt.text(750,  90, "Median : " + str(round(np.median(doc_lens))))
  plt.text(750,  80, "Stdev   : " + str(round(np.std(doc_lens))))
  plt.text(750,  70, "1%ile    : " + str(round(np.quantile(doc_lens, q=0.01))))
  plt.text(750,  60, "99%ile  : " + str(round(np.quantile(doc_lens, q=0.99))))
  
  plt.gca().set(xlim=(0, 1000), ylabel='Number of Documents', xlabel='Document Word Count')
  plt.tick_params(size=16)
  plt.xticks(np.linspace(0,1000,9))
  plt.title('Distribution of Document Word Counts', fontdict=dict(size=22))
  plt.show()
  
  cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'
  
  fig, axes = plt.subplots(2,2,figsize=(16,14), dpi=160, sharex=True, sharey=True)
  
  for i, ax in enumerate(axes.flatten()):    
      df_dominant_topic_sub = df_dominant_topic.loc[df_dominant_topic.Dominant_Topic == i, :]
      doc_lens = [len(d) for d in df_dominant_topic_sub.Text]
      ax.hist(doc_lens, bins = 1000, color=cols[i])
      ax.tick_params(axis='y', labelcolor=cols[i], color=cols[i])
      sns.kdeplot(doc_lens, color="black", shade=False, ax=ax.twinx())
      ax.set(xlim=(0, 1000), xlabel='Document Word Count')
      ax.set_ylabel('Number of Documents', color=cols[i])
      ax.set_title('Topic: '+str(i), fontdict=dict(size=16, color=cols[i]))
  
  fig.tight_layout()
  fig.subplots_adjust(top=0.90)
  plt.xticks(np.linspace(0,1000,9))
  fig.suptitle('Distribution of Document Word Counts by Dominant Topic', fontsize=22)
  plt.show()
  
  """###Word Clouds of Top N Keywords in Each Topic"""
  
  # 1. Wordcloud of Top N words in each topic
  from matplotlib import pyplot as plt
  !pip install wordcloud
  from wordcloud import WordCloud, STOPWORDS
  import matplotlib.colors as mcolors
  
  cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]  # more colors: 'mcolors.XKCD_COLORS'
  
  cloud = WordCloud(stopwords=stop_words,
                    background_color='white',
                    width=2500,
                    height=1800,
                    max_words=10,
                    colormap='tab10',
                    color_func=lambda *args, **kwargs: cols[i],
                    prefer_horizontal=1.0)
  
  topics = lda_model.show_topics(formatted=False)
  
  fig, axes = plt.subplots(2, 2, figsize=(10,10), sharex=True, sharey=True)
  
  for i, ax in enumerate(axes.flatten()):
      fig.add_subplot(ax)
      topic_words = dict(topics[i][1])
      cloud.generate_from_frequencies(topic_words, max_font_size=300)
      plt.gca().imshow(cloud)
      plt.gca().set_title('Topic ' + str(i), fontdict=dict(size=16))
      plt.gca().axis('off')
  
  
  plt.subplots_adjust(wspace=0, hspace=0)
  plt.axis('off')
  plt.margins(x=0, y=0)
  plt.tight_layout()
  plt.show()
  
  """###Word Counts of Topic Keywords"""
  
  topics = lda_model.show_topics(formatted=False)
  data_flat = [w for w_list in data_ready for w in w_list]
  counter = Counter(data_flat)
  
  out = []
  for i, topic in topics:
      for word, weight in topic:
          out.append([word, i , weight, counter[word]])
  
  df = pd.DataFrame(out, columns=['word', 'topic_id', 'importance', 'word_count'])        
  
  # Plot Word Count and Weights of Topic Keywords
  fig, axes = plt.subplots(2, 2, figsize=(16,10), sharey=True, dpi=160)
  cols = [color for name, color in mcolors.TABLEAU_COLORS.items()]
  for i, ax in enumerate(axes.flatten()):
      ax.bar(x='word', height="word_count", data=df.loc[df.topic_id==i, :], color=cols[i], width=0.5, alpha=0.3, label='Word Count')
      ax_twin = ax.twinx()
      ax_twin.bar(x='word', height="importance", data=df.loc[df.topic_id==i, :], color=cols[i], width=0.2, label='Weights')
      ax.set_ylabel('Word Count', color=cols[i])
      ax_twin.set_ylim(0, 0.030); ax.set_ylim(0, 3500)
      ax.set_title('Topic: ' + str(i), color=cols[i], fontsize=16)
      ax.tick_params(axis='y', left=False)
      ax.set_xticklabels(df.loc[df.topic_id==i, 'word'], rotation=30, horizontalalignment= 'right')
      ax.legend(loc='upper left'); ax_twin.legend(loc='upper right')
  
  fig.tight_layout(w_pad=2)    
  fig.suptitle('Word Count and Importance of Topic Keywords', fontsize=22, y=1.05)    
  plt.show()
  
  """###Sentence Chart Colored by Topic"""
  
  # Sentence Coloring of N Sentences
  from matplotlib.patches import Rectangle
  
  def sentences_chart(lda_model=lda_model, corpus=corpus, start = 0, end = 13):
      corp = corpus[start:end]
      mycolors = [color for name, color in mcolors.TABLEAU_COLORS.items()]
  
      fig, axes = plt.subplots(end-start, 1, figsize=(20, (end-start)*0.95), dpi=160)       
      axes[0].axis('off')
      for i, ax in enumerate(axes):
          if i > 0:
              corp_cur = corp[i-1] 
              topic_percs, wordid_topics, wordid_phivalues = lda_model[corp_cur]
              word_dominanttopic = [(lda_model.id2word[wd], topic[0]) for wd, topic in wordid_topics]    
              ax.text(0.01, 0.5, "Doc " + str(i-1) + ": ", verticalalignment='center',
                      fontsize=16, color='black', transform=ax.transAxes, fontweight=700)
  
              # Draw Rectange
              topic_percs_sorted = sorted(topic_percs, key=lambda x: (x[1]), reverse=True)
              ax.add_patch(Rectangle((0.0, 0.05), 0.99, 0.90, fill=None, alpha=1, 
                                     color=mycolors[topic_percs_sorted[0][0]], linewidth=2))
  
              word_pos = 0.06
              for j, (word, topics) in enumerate(word_dominanttopic):
                  if j < 14:
                      ax.text(word_pos, 0.5, word,
                              horizontalalignment='left',
                              verticalalignment='center',
                              fontsize=16, color=mycolors[topics],
                              transform=ax.transAxes, fontweight=700)
                      word_pos += .009 * len(word)  # to move the word for the next iter
                      ax.axis('off')
              ax.text(word_pos, 0.5, '. . .',
                      horizontalalignment='left',
                      verticalalignment='center',
                      fontsize=16, color='black',
                      transform=ax.transAxes)       
  
      plt.subplots_adjust(wspace=0, hspace=0)
      plt.suptitle('Sentence Topic Coloring for Documents: ' + str(start) + ' to ' + str(end-2), fontsize=22, y=0.95, fontweight=700)
      plt.tight_layout()
      plt.show()
  
  sentences_chart()
  
  """###What are the most discussed topics in the documents?"""
  
  # Sentence Coloring of N Sentences
  def topics_per_document(model, corpus, start=0, end=1):
      corpus_sel = corpus[start:end]
      dominant_topics = []
      topic_percentages = []
      for i, corp in enumerate(corpus_sel):
          topic_percs, wordid_topics, wordid_phivalues = model[corp]
          dominant_topic = sorted(topic_percs, key = lambda x: x[1], reverse=True)[0][0]
          dominant_topics.append((i, dominant_topic))
          topic_percentages.append(topic_percs)
      return(dominant_topics, topic_percentages)
  
  dominant_topics, topic_percentages = topics_per_document(model=lda_model, corpus=corpus, end=-1)            
  
  # Distribution of Dominant Topics in Each Document
  df = pd.DataFrame(dominant_topics, columns=['Document_Id', 'Dominant_Topic'])
  dominant_topic_in_each_doc = df.groupby('Dominant_Topic').size()
  df_dominant_topic_in_each_doc = dominant_topic_in_each_doc.to_frame(name='count').reset_index()
  
  # Total Topic Distribution by actual weight
  topic_weightage_by_doc = pd.DataFrame([dict(t) for t in topic_percentages])
  df_topic_weightage_by_doc = topic_weightage_by_doc.sum().to_frame(name='count').reset_index()
  
  # Top 3 Keywords for each Topic
  topic_top3words = [(i, topic) for i, topics in lda_model.show_topics(formatted=False) 
                                   for j, (topic, wt) in enumerate(topics) if j < 3]
  
  df_top3words_stacked = pd.DataFrame(topic_top3words, columns=['topic_id', 'words'])
  df_top3words = df_top3words_stacked.groupby('topic_id').agg(', \n'.join)
  df_top3words.reset_index(level=0,inplace=True)
  
  # Plot
  fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10, 4), dpi=120, sharey=True)
  
  # Topic Distribution by Dominant Topics
  ax1.bar(x='Dominant_Topic', height='count', data=df_dominant_topic_in_each_doc, width=.5, color='firebrick')
  ax1.set_xticks(range(df_dominant_topic_in_each_doc.Dominant_Topic.unique().__len__()))
  tick_formatter = FuncFormatter(lambda x, pos: 'Topic ' + str(x)+ '\n' + df_top3words.loc[df_top3words.topic_id==x, 'words'].values[0])
  ax1.xaxis.set_major_formatter(tick_formatter)
  ax1.set_title('Number of Documents by Dominant Topic', fontdict=dict(size=10))
  ax1.set_ylabel('Number of Documents')
  ax1.set_ylim(0, 1000)
  
  # Topic Distribution by Topic Weights
  ax2.bar(x='index', height='count', data=df_topic_weightage_by_doc, width=.5, color='steelblue')
  ax2.set_xticks(range(df_topic_weightage_by_doc.index.unique().__len__()))
  ax2.xaxis.set_major_formatter(tick_formatter)
  ax2.set_title('Number of Documents by Topic Weightage', fontdict=dict(size=10))
  
  plt.show()
  
  """### t-SNE Clustering Chart"""
  
  # Get topic weights and dominant topics ------------
  from sklearn.manifold import TSNE
  from bokeh.plotting import figure, output_file, show
  from bokeh.models import Label
  from bokeh.io import output_notebook
  
  # Get topic weights
  topic_weights = []
  for i, row_list in enumerate(lda_model[corpus]):
      topic_weights.append([w for i, w in row_list[0]])
  
  # Array of topic weights    
  arr = pd.DataFrame(topic_weights).fillna(0).values
  
  # Keep the well separated points (optional)
  arr = arr[np.amax(arr, axis=1) > 0.35]
  
  # Dominant topic number in each doc
  topic_num = np.argmax(arr, axis=1)
  
  # tSNE Dimension Reduction
  tsne_model = TSNE(n_components=2, verbose=1, random_state=0, angle=.99, init='pca')
  tsne_lda = tsne_model.fit_transform(arr)
  
  # Plot the Topic Clusters using Bokeh
  output_notebook()
  n_topics = 4
  mycolors = np.array([color for name, color in mcolors.TABLEAU_COLORS.items()])
  plot = figure(title="t-SNE Clustering of {} LDA Topics".format(n_topics), 
                plot_width=900, plot_height=700)
  plot.scatter(x=tsne_lda[:,0], y=tsne_lda[:,1], color=mycolors[topic_num])
  show(plot)
  
  """###pyLDAvis"""
  
  import pyLDAvis.gensim
  pyLDAvis.enable_notebook()
  vis = pyLDAvis.gensim.prepare(lda_model, corpus, dictionary=lda_model.id2word)
  vis
  