content: # This code is supporting material for the book
  # Building Machine Learning Systems with Python
  # by Willi Richert and Luis Pedro Coelho
  # published by PACKT Publishing
  #
  # It is made available under the MIT License
  
  from __future__ import print_function
  from wordcloud import create_cloud
  try:
      from gensim import corpora, models, matutils
  except:
      print("import gensim failed.")
      print()
      print("Please install it")
      raise
  
  import matplotlib.pyplot as plt
  import numpy as np
  from os import path
  
  NUM_TOPICS = 100
  
  # Check that data exists
  if not path.exists('./data/ap/ap.dat'):
      print('Error: Expected data to be present at data/ap/')
      print('Please cd into ./data & run ./download_ap.sh')
  
  # Load the data
  corpus = corpora.BleiCorpus('./data/ap/ap.dat', './data/ap/vocab.txt')
  
  # Build the topic model
  model = models.ldamodel.LdaModel(
      corpus, num_topics=NUM_TOPICS, id2word=corpus.id2word, alpha=None)
  
  # Iterate over all the topics in the model
  for ti in range(model.num_topics):
      words = model.show_topic(ti, 64)
      tf = sum(f for f, w in words)
      with open('topics.txt', 'w') as output:
          output.write('\n'.join('{}:{}'.format(w, int(1000. * f / tf)) for f, w in words))
          output.write("\n\n\n")
  
  # We first identify the most discussed topic, i.e., the one with the
  # highest total weight
  
  topics = matutils.corpus2dense(model[corpus], num_terms=model.num_topics)
  weight = topics.sum(1)
  max_topic = weight.argmax()
  
  
  # Get the top 64 words for this topic
  # Without the argument, show_topic would return only 10 words
  words = model.show_topic(max_topic, 64)
  
  # This function will actually check for the presence of pytagcloud and is otherwise a no-op
  create_cloud('cloud_blei_lda.png', words)
  
  num_topics_used = [len(model[doc]) for doc in corpus]
  fig,ax = plt.subplots()
  ax.hist(num_topics_used, np.arange(42))
  ax.set_ylabel('Nr of documents')
  ax.set_xlabel('Nr of topics')
  fig.tight_layout()
  fig.savefig('Figure_04_01.png')
  
  
  # Now, repeat the same exercise using alpha=1.0
  # You can edit the constant below to play around with this parameter
  ALPHA = 1.0
  
  model1 = models.ldamodel.LdaModel(
      corpus, num_topics=NUM_TOPICS, id2word=corpus.id2word, alpha=ALPHA)
  num_topics_used1 = [len(model1[doc]) for doc in corpus]
  
  fig,ax = plt.subplots()
  ax.hist([num_topics_used, num_topics_used1], np.arange(42))
  ax.set_ylabel('Nr of documents')
  ax.set_xlabel('Nr of topics')
  
  # The coordinates below were fit by trial and error to look good
  ax.text(9, 223, r'default alpha')
  ax.text(26, 156, 'alpha=1.0')
  fig.tight_layout()
  fig.savefig('Figure_04_02.png')
  
  