text: from __future__ import print_function
  
  
  from codecs import open
  from configparser import RawConfigParser as ConfigParser, NoOptionError
  import csv
  from datetime import datetime, timedelta
  from functools import partial
  from importlib import import_module
  from io import BytesIO,StringIO
  import json
  import itertools
  import math
  import os.path
  from pkg_resources import resource_filename
  import re
  import socket
  import sys
  from urllib import unquote
  import webbrowser
  
  from bottle import request, response, route, run, static_file, Bottle
  from lib.color import get_topic_colors, rgb2hex
  from lib.ssl import SSLWSGIRefServer
  from lib.util import (int_prompt, bool_prompt, is_valid_filepath,
      is_valid_configfile, get_static_resource_path)
  
  from vsm.corpus import Corpus
  from vsm.model.lda import LDA
  from vsm.viewer.ldacgsviewer import LdaCgsViewer as LDAViewer
  from vsm.viewer.wrappers import doc_label_name
  
  import random
  import pystache
  import sys
  reload(sys)  
  sys.setdefaultencoding('utf8')
  __all__ = ['populate_parser', 'main', '_set_acao_headers', 'Application']
      
  
  def _set_acao_headers(f):
      """
      Decorator to set Access-Control-Allow-Origin headers to enable cross-InPhO
      embedding of Topic Explorer bars.
      """
      def set_header(*args, **kwargs):
          host = request.get_header('Origin')
          if host and 'cogs.indiana.edu' in host: # pragma: no cover
              response.headers['Access-Control-Allow-Origin'] = host
          elif host and 'codepen.io' in host:
              response.headers['Access-Control-Allow-Origin'] = host
          return f(*args, **kwargs)
      return set_header
  
  
  def _cache_date(days=1):
      """
      Helper function to return the date for the cache header.
      """
      time = datetime.now() + timedelta(days=days)
      return time.strftime("%a, %d %b %Y %I:%M:%S GMT")
  
  
  class Application(Bottle):
      """
      This is the primary Bottle application for the Topic Explorer.
      Each Application corresponds to a single Corpus object, but may
      have multiple LDA model objects.
      """
  
      def __init__(self, corpus_file='', model_pattern='', topic_range=None,
                   context_type='', label_module=None, config_file='',
                   fulltext=False, corpus_path='', **kwargs):
          super(Application, self).__init__()
  
          self.config_file = config_file
  
          # setup routes
          self.renderer = pystache.Renderer(escape=lambda u: u,
              string_encoding='utf8')
          self.icons = kwargs.get('icons', 'link')
          if fulltext:
              self._serve_fulltext(corpus_path)
          self._setup_routes(**kwargs)
  
          # load corpus
          self.context_type = context_type
          self.label_name = self.context_type + '_label'
          self._load_label_module(label_module, config_file)
          cwd = os.getcwd()
          print (cwd+corpus_file)
          self._load_corpus(cwd+corpus_file)
  
          # load viewers
          self.v = dict()
          self.topic_range = topic_range
          self.colors = dict()
          self._load_viewers(cwd+model_pattern)
  
      def _load_label_module(self, label_module, config_file):
          try:
              label_module = import_module(label_module)
              print("imported label module")
              label_module.init(self, config_file)
          except (ImportError, NoOptionError, AttributeError):
              pass
  
          try:
              self.label = label_module.label
              print("imported label function")
          except (AttributeError, UnboundLocalError):
              self.label = lambda x: x
              print("using default label function")
  
          try:
              self.id_fn = label_module.id_fn
              print("imported id function")
          except (AttributeError, UnboundLocalError):
              self.id_fn = lambda metadata: metadata[self.label_name]
              print("using default id function")
  
      def _load_corpus(self, corpus_file):
          self.c = Corpus.load(corpus_file)
          self.labels = self.c.view_metadata(self.context_type)[self.label_name]
  
      def _load_viewers(self, model_pattern):
          self.id_fn = lambda md: md[self.label_name]
          for k in self.topic_range:
              m = LDA.load(model_pattern.format(k))
              self.v[k] = LDAViewer(self.c, m)
              self.colors[k] = dict(get_topic_colors(self.v[k]))
              self.v[k].dist_doc_doc = partial(
                  self.v[k].dist_doc_doc, label_fn=self.id_fn)
              self.v[k].dist_top_doc = partial(
                  self.v[k].dist_top_doc, label_fn=self.id_fn)
  
      def _setup_routes(self, **kwargs):
          @self.route('/<k:int>/doc_topics/<doc_id>')
          @_set_acao_headers
          def doc_topic_csv(k, doc_id):
              response.content_type = 'text/csv; charset=UTF8'
  
              data = self.v[k].doc_topics(doc_id)
  
              if sys.version_info[0] == 3:
                  output = StringIO()
              else:
                  output = BytesIO()
  
              writer = csv.writer(output)
              writer.writerow(['topic', 'prob'])
              writer.writerows([(t, "%6f" % p) for t, p in data])
  
              return output.getvalue()
  
          @self.route('/<k:int>/docs/<doc_id>')
          @_set_acao_headers
          def doc_csv(k, doc_id, threshold=0.2):
              response.content_type = 'text/csv; charset=UTF8'
  
              data = self.v[k].dist_doc_doc(doc_id)
  
              if sys.version_info[0] == 3:
                  output = StringIO()
              else:
                  output = BytesIO()
  
              writer = csv.writer(output)
              writer.writerow(['doc', 'prob'])
              writer.writerows([(d, "%6f" % p) for d, p in data if p > threshold])
  
              return output.getvalue()
  
          @self.route('/<k:int>/topics/<topic_no:int>.json')
          @_set_acao_headers
          def topic_json(k, topic_no, N=40):
              response.content_type = 'application/json; charset=UTF8'
              try:
                  N = int(request.query.n)
              except:
                  pass
  
              if N > 0:
                  data = self.v[k].dist_top_doc([topic_no])[:N]
              else:
                  data = self.v[k].dist_top_doc([topic_no])[N:]
                  data = reversed(data)
  
              docs = [doc for doc, prob in data]
              doc_topics_mat = self.v[k].doc_topics(docs)
              docs = self.get_docs(docs, id_as_key=True)
  
              js = []
              for doc_prob, topics in zip(data, doc_topics_mat):
                  doc, prob = doc_prob
                  struct = docs[doc]
                  struct.update({'prob': float(1 - prob),
                                 'topics': dict([(str(t), float(p)) for t, p in topics])})
                  js.append(struct)
  
              return json.dumps(js)
  
          @self.route('/<k:int>/docs_topics/<doc_id:path>.json')
          @_set_acao_headers
          def doc_topics(k, doc_id, N=40):
              try:
                  N = int(request.query.n)
              except:
                  pass
  
              response.content_type = 'application/json; charset=UTF8'
  
              if N > 0:
                  data = self.v[k].dist_doc_doc(doc_id)[:N]
              else:
                  data = self.v[k].dist_doc_doc(doc_id)[N:]
                  data = reversed(data)
  
              docs = [doc for doc, prob in data]
              doc_topics_mat = self.v[k].doc_topics(docs)
              docs = self.get_docs(docs, id_as_key=True)
  
              js = []
              for doc_prob, topics in zip(data, doc_topics_mat):
                  doc, prob = doc_prob
                  struct = docs[doc]
                  struct.update({'prob': float(1 - prob),
                                 'topics': dict([(str(t), float(p)) for t, p in topics])})
                  js.append(struct)
  
              return json.dumps(js)
  
          @self.route('/<k:int>/word_docs.json')
          @_set_acao_headers
          def word_docs(k, N=40):
              import numpy as np
              try:
                  N = int(request.query.n)
              except:
                  pass
              try:
                  query = request.query.q.lower()
                  if self.c.words.dtype.type == np.string_:
                      query = query.encode('ascii', 'ignore')
                  
                  query = query.split('|')
              except:
                  raise Exception('Must specify a query')
  
              response.content_type = 'application/json; charset=UTF8'
  
              query = [word for word in query if word in self.c.words]
  
              # abort if there are no terms in the query
              if not query:
                  response.status = 400  # Bad Request
                  return "Search terms not in model"
  
              topics = self.v[k].dist_word_top(query, show_topics=False)
              data = self.v[k].dist_top_doc(topics['i'],
                                            weights=(topics['value'].max() - topics['value']))
  
              if N > 0:
                  data = data[:N]
              else:
                  data = data[N:]
                  data = reversed(data)
  
              docs = [doc for doc, prob in data]
              doc_topics_mat = self.v[k].doc_topics(docs)
              docs = self.get_docs(docs, id_as_key=True)
  
              js = []
              for doc_prob, topics in zip(data, doc_topics_mat):
                  doc, prob = doc_prob
                  struct = docs[doc]
                  struct.update({'prob': float(1 - prob),
                                 'topics': dict([(str(t), float(p)) for t, p in topics])})
                  js.append(struct)
  
              return json.dumps(js)
  
          @self.route('/<k:int>/topics.json')
          @_set_acao_headers
          def topics(k):
              from lib.color import rgb2hex
  
              response.content_type = 'application/json; charset=UTF8'
              response.set_header('Expires', _cache_date())
              response.set_header('Cache-Control', 'max-age=86400')
              
              # populate word values
              data = self.v[k].topics()
  
              js = {}
              wordmax = 10  # for alphabetic languages
              if kwargs.get('lang', None) == 'cn':
                  wordmax = 25  # for ideographic languages
  
              for i, topic in enumerate(data):
                  js[str(i)] = {
                      "color": rgb2hex(self.colors[k][i]),
                      'words': dict([(str(w), float(p))
                                         for w, p in topic[:wordmax]])
                      }
  
              return json.dumps(js)
  
          @self.route('/topics.json')
          @_set_acao_headers
          def word_topic_distance():
              import numpy as np
              response.content_type = 'application/json; charset=UTF8'
  
              # parse query
              try:
                  if '|' in request.query.q:
                      query = request.query.q.lower()
                      if self.c.words.dtype.type == np.string_:
                          query = query.encode('ascii', 'ignore')
                  
                      query = query.split('|')
                  else:
                      query = request.query.q.lower()
                      if self.c.words.dtype.type == np.string_:
                          query = query.encode('ascii', 'ignore')
                  
                      query = query.split(' ')
              except:
                  raise Exception('Must specify a query')
  
              stopped_words = [word for word in query 
                                   if word in self.c.stopped_words]
              query = [word for word in query if word in self.c.words]
  
              # abort if there are no terms in the query
              if not query and stopped_words:
                  response.status = 410  # Gone
                  return "Search terms removed using stoplist: " + ' '.join(query)
              elif not query:
                  response.status = 404  # Not Found
                  return "Search terms not in corpus"
  
  
              # calculate distances
              distances = dict()
              for k, viewer in self.v.items():
                  d = viewer.dist_word_top(query, show_topics=False)
                  distances[k] = np.fromiter(
                      ((k, row['i'], row['value']) for row in d),
                      dtype=[('k', '<i8'), ('i', '<i8'), ('value', '<f8')])
  
              # merge and sort all topics across all models
              merged_similarity = np.hstack(list(distances.values()))
              sorted_topics = merged_similarity[np.argsort(merged_similarity['value'])]
  
              # return data
              data = [{'k' : int(t['k']),
                       't' : int(t['i']),
                       'distance' : float(t['value']) } for t in sorted_topics]
              return json.dumps(data)
  
  
          @self.route('/topics')
          @_set_acao_headers
          def view_clusters():
              with open(get_static_resource_path('www/master.mustache.html'),
                        encoding='utf-8') as tmpl_file:
                  template = tmpl_file.read()
  
              tmpl_params = {'body' : _render_template('cluster.mustache.html'),
                             'topic_range': self.topic_range}
              return self.renderer.render(template, tmpl_params)
  
  
          @self.route('/docs.json')
          @_set_acao_headers
          def docs(docs=None, q=None, n=None):
              response.content_type = 'application/json; charset=UTF8'
              response.set_header('Expires', _cache_date())
  
              try:
                  if request.query.q:
                      q = request.query.q
                      n = 10
              except:
                  pass
  
              try:
                  if request.query.id:
                      docs = [request.query.id]
              except:
                  pass
  
              try:
                  if request.query.random:
                      response.set_header('Expires', 0)
                      response.set_header('Pragma', 'no-cache')
                      response.set_header('Cache-Control', 'no-cache, no-store, must-revalidate')
                      docs = [random.choice(self.labels)]
              except:
                  pass
  
              js = self.get_docs(docs, query=q, n=n)
  
              return json.dumps(js)
  
          @self.route('/icons.js')
          def icons():
              with open(get_static_resource_path('www/icons.js')) as icons:
                  text = '{0}\n var icons = {1};'\
                      .format(icons.read(), json.dumps(self.icons))
              return text
  
          def _render_template(page):
              response.set_header('Expires', _cache_date())
  
              with open(get_static_resource_path('www/' + page),
                        encoding='utf-8') as tmpl_file:
                  template = tmpl_file.read()
  
              tmpl_params = {'corpus_name': kwargs.get('corpus_name', ''),
                             'corpus_link': kwargs.get('corpus_link', ''),
                             'context_type': self.context_type,
                             'topic_range': self.topic_range,
                             'doc_title_format': kwargs.get('doc_title_format', '{0}'),
                             'doc_url_format': kwargs.get('doc_url_format', ''),
                             'home_link': kwargs.get('home_link', '/')}
              return self.renderer.render(template, tmpl_params)
  
          @self.route('/<k:int>/')
          def index(k):
              with open(get_static_resource_path('www/master.mustache.html'),
                        encoding='utf-8') as tmpl_file:
                  template = tmpl_file.read()
  
              tmpl_params = {'body' : _render_template('bars.mustache.html'),
                             'topic_range': self.topic_range}
              return self.renderer.render(template, tmpl_params)
  
          @self.route('/cluster.csv')
          @_set_acao_headers
          def cluster_csv(second=False):
              filename = kwargs.get('cluster_path')
              print("Retrieving cluster.csv:", filename)
              if not filename or not os.path.exists(filename):
                  import topicexplorer.train
                  filename = topicexplorer.train.cluster(10, self.config_file)
                  kwargs['cluster_path'] = filename
  
              root, filename = os.path.split(filename)
              return static_file(filename, root=root)
          
          @self.route('/description.md')
          @_set_acao_headers
          def description():
              filename = kwargs.get('corpus_desc')
              if not filename:
                  response.status = 404
                  return "File not found"
              filename = get_static_resource_path(filename)
              root, filename = os.path.split(filename)
              return static_file(filename, root=root)
          
          @self.route('/')
          @_set_acao_headers
          def cluster():
              with open(get_static_resource_path('www/master.mustache.html'),
                        encoding='utf-8') as tmpl_file:
                  template = tmpl_file.read()
  
              tmpl_params = {'body' : _render_template('splash.mustache.html'),
                             'topic_range': self.topic_range}
              return self.renderer.render(template, tmpl_params)
  
          @self.route('/<filename:path>')
          @_set_acao_headers
          def send_static(filename):
              return static_file(filename, root=get_static_resource_path('www/'))
  
      def _serve_fulltext(self, corpus_path):
          @self.route('/fulltext/<doc_id:path>')
          @_set_acao_headers
          def get_doc(doc_id):
              pdf_path = os.path.join(corpus_path, re.sub('txt$', 'pdf', doc_id))
              if os.path.exists(pdf_path.encode('utf-8')):
                  doc_id = re.sub('txt$', 'pdf', doc_id)
              if os.path.exists(os.path.join(corpus_path, doc_id + '.txt')):
                  doc_id = doc_id + '.txt'
              # here we deal with case where corpus_path and doc_id overlap
              (fdirs, lastdir) = os.path.split(corpus_path)
              if re.match('^' + lastdir, doc_id):
                  return static_file(doc_id, root=fdirs)
              else:
                  return static_file(doc_id, root=corpus_path)
  
      def get_docs(self, docs=None, id_as_key=False, query=None, n=None):
          ctx_md = self.c.view_metadata(self.context_type)
  
          if docs:
              # filter to metadata for selected docs
              ids = [self.c.meta_int(self.context_type, {self.label_name: doc}) for doc in docs]
              ctx_md = ctx_md[ids]
          else:
              # get metadata for all documents
              docs = self.labels
  
          js = dict() if id_as_key else list()
  
          for doc, md in zip(docs, ctx_md):
              if query is None or query.lower() in self.label(doc).lower():
                  struct = {
                      'id': doc,
                      'label': self.label(doc)
                  }
                  # TODO: fix metadata login
                      #'metadata': dict(zip(md.dtype.names, (str(m) for m in md)))}
                  if id_as_key:
                      js[doc] = struct
                  else:
                      js.append(struct)
                  if n is not None:
                      n = n-1
                      if n <= 0:
                          break
  
  
          return js
  
  
  def get_host_port(args):
      """
      Returns the hostname and port number
      """
      config = ConfigParser({'port': '3001', 'host': '0.0.0.0'})
      with open(args.config, encoding='utf8') as configfile:
          config.read_file(configfile)
  
      # automatic port assignment
      def test_port(port):
          try:
              host = args.host or config.get("www", "host")
              if host == '0.0.0.0':
                  host = 'localhost'
              try:
                  s = socket.create_connection((host, port), 2)
                  s.close()
                  raise IOError("Socket connectable on port {0}".format(port))
              except socket.error:
                  pass
              return port
          except IOError:
              if not args.quiet:
                  port = int_prompt(
                      "Conflict on port {0}. Enter new port:".format(port))
                  return test_port(port)
              else:
                  raise IOError(
                      "Conflict on port {0}. Try running with -p to manually set new port.".format(port))
  
      port = args.port or int(config.get('www', 'port').format(0))
      port = test_port(port)
  
      # prompt to save
      if (int(config.get("www", "port").format(0))) != port:
          if not args.quiet and bool_prompt(
              "Change default baseport to {0}?".format(port), default=True):
              config.set("www", "port", str(port))
  
              # create deep copy of configuration
              # see http://stackoverflow.com/a/24343297
              config_string = StringIO()
              config.write(config_string)
  
              # skip DEFAULT section
              config_string.seek(0)
              idx = config_string.getvalue().index("[main]")
              config_string.seek(idx)
  
              # read deep copy
              new_config = ConfigParser()
              config.read_file(config_string)
  
              # write deep copy without DEFAULT section
              # this preserves DEFAULT for rest of program
              with open(args.config, 'w') as configfh:
                  new_config.write(configfh)
  
      # hostname assignment
      host = args.host or config.get('www', 'host')
  
      return host, port
  
  
  def main(args, app=None):
      if app is None:
          app = create_app(args)
  
      host, port = get_host_port(args)
  
      if args.browser:
          if host == '0.0.0.0':
              link_host = socket.gethostname()
          else:
              link_host = host
          url = "http://{host}:{port}/"
          url = url.format(host=link_host, port=port, k=min(app.topic_range))
          webbrowser.open(url)
  
          print("TIP: Browser launch can be disabled with the '--no-browser' argument:")
          print("topicexplorer serve --no-browser", args.config, "\n")
  
      app.run(server='paste', host=host, port=port)
  
  
  def create_app(args):
      # load in the configuration file
      config = ConfigParser({
          'certfile': None,
          'keyfile': None,
          'ca_certs': None,
          'ssl': False,
          'port': '3001',
          'host': '0.0.0.0',
          'icons': 'link',
          'corpus_link': None,
          'doc_title_format': '{0}',
          'doc_url_format': '',
          'raw_corpus': None,
          'label_module': None,
          'fulltext': 'false',
          'topics': None,
          'cluster': None,
          'corpus_desc' : None,
          'home_link' : '/',
          'lang': None})
  
      with open(args.config, encoding='utf8') as configfile:
          config.read_file(configfile)
  
      # path variables
      context_type = config.get('main', 'context_type')
      corpus_file = config.get('main', 'corpus_file')
      model_pattern = config.get('main', 'model_pattern')
      cluster_path = config.get('main', 'cluster')
  
      # language customization
      lang = config.get('main', 'lang')
  
      # set topic_range
      if config.get('main', 'topics'):
          topic_range = eval(config.get('main', 'topics'))
  
      # get icons_list
      config_icons = config.get('www', 'icons').split(",")
      if args.fulltext or config.getboolean('www', 'fulltext'):
          if not any('fulltext' in icon for icon in config_icons):
              config_icons.insert(0, 'fulltext-inline')
  
      # Create application object
      corpus_name = config.get('www', 'corpus_name')
      corpus_link = config.get('www', 'corpus_link')
      doc_title_format = config.get('www', 'doc_title_format')
      doc_url_format = config.get('www', 'doc_url_format')
      home_link = config.get('www', 'home_link')
      label_module = config.get('main', 'label_module')
      corpus_path = config.get('main', 'raw_corpus')
      corpus_desc = config.get('main', 'corpus_desc')
      fulltext = args.fulltext or config.getboolean('www', 'fulltext')
  
      app = Application(corpus_file=corpus_file,
                        model_pattern=model_pattern,
                        topic_range=topic_range,
                        context_type=context_type,
                        label_module=label_module,
                        config_file=args.config,
                        corpus_path=corpus_path,
                        fulltext=fulltext,
                        lang=lang,
                        icons=config_icons,
                        corpus_name=corpus_name,
                        corpus_link=corpus_link,
                        doc_title_format=doc_title_format,
                        doc_url_format=doc_url_format,
                        cluster_path=cluster_path,
                        corpus_desc=corpus_desc,
                        home_link=home_link)
  
      """
      host, port = get_host_port(args) 
      """
      # app.run(host='0.0.0.0', port=8081)
      return app
  
  
  def populate_parser(parser):
      parser.add_argument('config', type=lambda x: is_valid_configfile(parser, x),
                          help="Configuration file path")
      parser.add_argument('-k', type=int, required=False,
                          help="Number of Topics")
      parser.add_argument('-p', dest='port', type=int,
                          help="Port Number", default=None)
      parser.add_argument('--host', default=None, help='Hostname')
      parser.add_argument('--no-browser', dest='browser', action='store_false')
      parser.add_argument("-q", "--quiet", action="store_true")
      parser.add_argument('--fulltext', action='store_true',
                          help='Serve raw corpus files.')
      parser.add_argument('--bibtex', default=None,
                          type=lambda x: is_valid_filepath(parser, x),
                          help='BibTeX library location')
      parser.add_argument('--ssl', action='store_true',
                          help="Use SSL (must specify certfile, keyfile, and ca_certs in config)")
      parser.add_argument('--ssl-certfile', dest='certfile', nargs="?",
                          const='server.pem', default=None,
                          type=lambda x: is_valid_filepath(parser, x),
                          help="SSL certificate file")
      parser.add_argument('--ssl-keyfile', dest='keyfile', default=None,
                          type=lambda x: is_valid_filepath(parser, x),
                          help="SSL certificate key file")
      parser.add_argument('--ssl-ca', dest='ca_certs', default=None,
                          type=lambda x: is_valid_filepath(parser, x),
                          help="SSL certificate authority file")
  
  if __name__ == '__main__':
      from argparse import ArgumentParser
  
      # argument parsing
      parser = ArgumentParser()
      populate_parser(parser)
      args = parser.parse_args()
  
      main(args)
  