text: # Pedram Daee <<pedram.daee@aalto.fi>>.
  # A DataLoader is an object that loads the log data from the first phase of the user study
  # A data matrix is a matrix with n rows (num_data) and d columns (num_features)
  # the features are themselves grouped in different views (num_views)
  
  import numpy as np
  from matplotlib import pyplot
  from gensim import corpora, models, similarities
  import json
  
  class DataLoader:
  
      def __init__(self, data_dir):
          """For initialization"""
          #Parameters
  
          #self.W2V = models.Word2Vec.load(data_dir+'/word2vec')
          self.corpus = corpora.MmCorpus(data_dir+'/corpus.mm')
          self.dictionary = corpora.Dictionary.load(data_dir+'/dictionary.dict')
          #self.index = similarities.MatrixSimilarity.load(data_dir+'/corpus.mm.index')
          #views_ind is a num_featurs*1 int array, e.g. [1,1,2,3] for 3 views and 4 features
          self.views_ind = np.load(data_dir+'/views_ind_1.npy')
          #views_ind = np.zeros(self.num_features)
  
          self.num_features = self.corpus.num_terms          #total number of features
          self.num_data = self.corpus.num_docs               #total number of data (snapshots)
          self.num_views = max(self.views_ind) + 1           #total number of views views= [0=BOW, 1=KW, 2=App, 3=People]
          self.Data = None                                   #todo: (it may be impossible to create this) a num_data*num_featurs array
          #name of the features
          self.feature_names = [self.dictionary.get(i) for i in range(self.num_features) ]
          self.num_items_per_view = [sum(self.views_ind == i) for i in range(self.num_views)]
  
      def print_info(self):
          print 'The corpus has %d items' %self.num_data+' and %d features'%self.num_features+\
                ' and %d views' %self.num_views +' there are %d' %self.corpus.num_nnz +' non-zero elements'
          print 'People view %d' % self.num_items_per_view[3]+ ' items, Application view %d' %self.num_items_per_view[2]+\
                ' items, KW view %d' %self.num_items_per_view[1]+' items, BOW view %d' %self.num_items_per_view[0]+' items.'
  
      def process_item_info(self):
          #This function is used for offline feedback gathering
          print 'The corpus has %d items' %self.num_data+' and %d features'%self.num_features+\
                ' and %d views' %self.num_views +' there are %d' %self.corpus.num_nnz +' non-zero elements'
          print 'People view %d' %sum(self.views_ind == 3)+ ' items, Application view %d' %sum(self.views_ind == 2)+\
                ' items, KW view %d' %sum(self.views_ind == 1)+' items, BOW view %d' %sum(self.views_ind == 0)+' items.'
  
          #get the document frequency of the terms (i.e. how many document did a particular term occur in):
          term_frequency_dic = self.dictionary.dfs
          sorted_term_ferequency = sorted(term_frequency_dic.iteritems(), key=lambda (k,v): (v,k), reverse=True)
          sorted_IDs = [sorted_term_ferequency[i][0] for i in range(self.num_features)]
  
          count_term = [y for (x,y) in sorted_term_ferequency]
          pyplot.hist(count_term[9000:self.num_features-1], 20, facecolor='green')
          pyplot.xlabel('number of occurrences in the corpus')
          pyplot.ylabel('count')
          pyplot.show()
          num_of_1_occurance = len([y for (x,y) in sorted_term_ferequency if y==1])
          print '%d terms' %num_of_1_occurance+' have only appeared once in the corpus'
          term_names_1_occurance = [(self.feature_names[x]) for (x,y) in sorted_term_ferequency if y==1 ]
          with open('term_names_1_occurance.txt', 'w') as outfile:
              json.dump(term_names_1_occurance, outfile)
          #those terms can be removed from the dictionary.todo: HOWEVER, they should be removed when the corpus is being made
          #print self.dictionary
          #self.dictionary.filter_extremes(no_below=2)
          #print self.dictionary
  
          AP_names = [(self.feature_names[sorted_IDs[i]]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 2]
          AP_ids = [(sorted_IDs[i]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 2]
  
          KW_names = [(self.feature_names[sorted_IDs[i]]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 1]
          KW_ids = [(sorted_IDs[i]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 1]
  
          People_names = [(self.feature_names[sorted_IDs[i]]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 3]
          People_ids = [(sorted_IDs[i]) for i in range(self.num_features) \
                      if  self.views_ind[sorted_IDs[i]] == 3]
  
          num_to_show = 1000 #
          data = {}
          data["AP_names"] = AP_names[:num_to_show]
          data["KW_names"] = KW_names[:num_to_show]
          data["People_names"] = People_names[:num_to_show]
          data["AP_ids"] = AP_ids[:num_to_show]
          data["KW_ids"] = KW_ids[:num_to_show]
          data["People_ids"] = People_ids[:num_to_show]
  
  
          with open('for_vuong.txt', 'w') as outfile:
              json.dump(data, outfile)
  