text: import json
  import collections 
  import sys
  import operator
  import math
  import psutil
  
  from collections import Counter
  from datetime import datetime, timedelta
  from operator import itemgetter
  from tqdm import tqdm
  
  class ExtendedCounter(Counter):
      def __mul__(self, ext):
          ext_words = self.copy()
          for k, v in ext_words.items():
              ext_words[k] = v * ext
          return ext_words
  
      __rmul__ = __mul__
  
  def tversky_params(reviewers, commit_count_words):
      d1 = sum((reviewers - commit_count_words).values()) 
      d2 = sum((commit_count_words - reviewers).values()) 
      i = sum((commit_count_words & reviewers).values())
  
      return i, d1, d2
  
  def callculate_tversky_for_reviewers(reviewers, commit_count_words, reviewers_last_date, last_if_absent, current, change_ext_function):
      top = collections.defaultdict(list)
  
      for r in reviewers:
          last = reviewers_last_date.get(r, last_if_absent)
          ext = change_ext_function(last, current)
          ext_words = ExtendedCounter(commit_count_words) * ext
  
          i, d1, d2 =  tversky_params( reviewers[r], ext_words)
          a = 0.0
          m_max = i / ( i + a * d1 + (1-a) * d2)
          top[m_max].append(r)
  
      sorted_top = sorted(top.keys(), reverse=True)
  
      return top, sorted_top
  
  def sorted_list_by_date(users, users_last_date):
      selected_users_dates = {u:users_last_date[u] for u in users}
  
      out = sorted(selected_users_dates.items(), key=operator.itemgetter(1), reverse=True)
      return [o[0] for o in out]
  
  def get_top_by_date(sorted_top, top, users_last_date):
      ret_top = collections.defaultdict(list)
      t_ret_top = []
  
      max_top = len(sorted_top)
  
      # check if sorted_top not empty and if top1 value >= 0
      if len(sorted_top) == 0 or sorted_top[0] == 0: 
          return ret_top
      
      # Callculate top1..top10, 
      for i in range(0, max_top):
          # use current top if available
          if i < max_top and sorted_top[i] > 0 : 
              l = sorted_list_by_date(top[sorted_top[i]], users_last_date)
              t_ret_top.extend(l)
  
      for i in range(1, 11):
          # reuse previous top
          ret_top[i].extend(t_ret_top[:i])
  
      return ret_top
  
  def get_top(sorted_top, top):
      ret_top = collections.defaultdict(list)
  
      max_top = len(sorted_top)
  
      # check if sorted_top not empty and if top1 value >= 0
      if len(sorted_top) == 0 or sorted_top[0] == 0: 
          return ret_top
      
      ret_top[1].extend(top[sorted_top[0]])
      
      # Callculate top1..top10, 
      for i in range(1, 10):
          # use current top if available
          if i < max_top and sorted_top[i] > 0 : 
              ret_top[i+1].extend(top[sorted_top[i]])
          # reuse previous top
          ret_top[i+1].extend(ret_top[i])
      
      return ret_top
  
  
  def callculate_commit_files_to_words(files):
      commit_count_words = ExtendedCounter()
  
      for f in files:
          commit_count_words.update(f.split("/"))
  
      return commit_count_words
  
  
  def parse_file(f_in):
      with open(f_in) as f: data = [json.loads(line) for line in f]
  
      data = sorted(data, key=itemgetter('changeId')) 
  
      prediction_id = collections.Counter()
      reviewers_id = collections.defaultdict(collections.Counter)
      reviewers_last_id = collections.defaultdict(None)
      suggested_reviewers_count_id = collections.defaultdict(list)
      mrr_sum_id = 0.0
      mrr_count_id = 0.0
  
  
      prediction_date = collections.Counter()
      reviewers_date = collections.defaultdict(collections.Counter)
      reviewers_last_date = collections.defaultdict(None)
      suggested_reviewers_count_date = collections.defaultdict(list)
      mrr_sum_date = 0.0
      mrr_count_date = 0.0
  
      review_count = 0.0
  
  
      for index,d in enumerate(tqdm(data)):
          
          commit_count_words = callculate_commit_files_to_words(d["files"])
  
          default_last_id = d['changeId'] -1
          current_id = d['changeId']
          top_id, sorted_top_id = callculate_tversky_for_reviewers(reviewers_id, commit_count_words, reviewers_last_id, default_last_id, current_id, change_id_ext) 
          top_dict_id = get_top_by_date(sorted_top_id, top_id, reviewers_last_id)
  
          default_last_date = convert_date(d['close_date']) - timedelta(days=1)
          current_date = convert_date(d['close_date']) 
          top_date, sorted_top_date = callculate_tversky_for_reviewers(reviewers_date, commit_count_words, reviewers_last_date, default_last_date, current_date, change_date_ext)
          top_dict_date = get_top_by_date(sorted_top_date, top_date, reviewers_last_date)
  
          for k in top_dict_id:
              suggested_reviewers_count_id[k].append(len(top_dict_id[k]))
              for hist in d["approve_history"]:
                  if  hist['userId'] in top_dict_id[k]: 
                      prediction_id[k] += 1
                      break
          for k in top_dict_date:
              suggested_reviewers_count_date[k].append(len(top_dict_date[k]))
              for hist in d["approve_history"]:
                  if  hist['userId'] in top_dict_date[k]:
                      prediction_date[k] += 1
                      break
  
          for hist in d["approve_history"]:
              last_id = reviewers_last_id.get(hist['userId'], d['changeId'] -1)
              current_id = d['changeId']
              ext_id = change_id_ext(last_id, current_id)
              reviewers_id[hist['userId']] += commit_count_words * ext_id
              reviewers_last_id[hist['userId']] = d['changeId']
  
              last_date = reviewers_last_date.get(hist['userId'], convert_date(hist['grant_date']) - timedelta(days=1))
              current_date = convert_date(hist['grant_date'])
              ext_date = change_date_ext(last_date, current_date)
              reviewers_date[hist['userId']] += commit_count_words * ext_date
              reviewers_last_date[hist['userId']] = convert_date(hist['grant_date'])
  
              reviewer = hist['userId']
              in_mrr_id = False
              for k in top_dict_id:
                  if reviewer in top_dict_id[k]: 
                      if not(in_mrr_id):
                          mrr_sum_id += 1.0 / k
                          mrr_count_id += 1
                          in_mrr_id = True
  
              in_mrr_date = False
              for k in top_dict_date:
                  if reviewer in top_dict_date[k]: 
                      if not(in_mrr_date):
                          mrr_sum_date += 1.0 / k
                          mrr_count_date += 1
                          in_mrr_date = True
              review_count += 1
  
      precision_id = collections.Counter()
      recall_id = collections.Counter()
  
      precision_date = collections.Counter()
      recall_date = collections.Counter()
  
      for key, value in prediction_id.items():
          precision_id[key] = float(value) / sum(i for i in suggested_reviewers_count_id[key])
          recall_id[key] = float(value) / review_count
          prediction_id[key] = float(value) / review_count
  
      for key, value in prediction_date.items():
          precision_date[key] = float(value) / sum(i for i in suggested_reviewers_count_date[key])
          recall_date[key] = float(value) / review_count
          prediction_date[key] = float(value) / review_count
  
  
      print("Id ext")
      for p in sorted(prediction_id): 
          print("Top %d = %f" % (p, float(prediction_id[p])))
      print("MRR %f" % (mrr_sum_id / mrr_count_id))
      print_precision(precision_id)
      print_recall(recall_id)
      print("Date ext")
      for p in sorted(prediction_date): 
          print("Top %d = %f" % (p, float(prediction_date[p])))
      print("MRR %f" % (mrr_sum_date / mrr_count_date))
      print_precision(precision_date)
      print_recall(recall_date)
      
      current_process = psutil.Process()
      current_memory_info = current_process.memory_info()
      print(current_memory_info)
   
  
  def print_precision(precision_top):
      print("Precision")
      for n in sorted(precision_top): 
          print("%f" % (float(precision_top[n])))
  
  def print_recall(recall_top):
      print("Recall")
      for n in sorted(recall_top): 
          print("%f" % (float(recall_top[n])))
  
  def convert_date(date_as_string):
      return datetime.strptime(date_as_string[:-3], "%Y-%m-%d %H:%M:%S.%f")
  
  def change_id_ext(last, current):
      half_life = 2500 
      # 1/2 after 2000 reviews
      fraction = 0.5
      
      difference = current - last
      
      decay = math.pow(math.pow(fraction, 1.0 / float(half_life)), difference)
  
      return 1.0 / decay
  
  def change_date_ext(last, current):
      half_life = 183 
      # 1/2 after 183 days
      fraction = 0.5
      
      difference = float((current - last).days)
      
      decay = math.pow(math.pow(fraction, 1.0 / float(half_life)), difference)
      return 1.0 / decay
  
  
  for f in sys.argv[1:]:
      users = parse_file(f)
  
  