text: # -*- coding: utf-8 -*-
  import codecs
  import numpy as np
  # load data of zhihu
  import os
  import pickle
  
  from keras.preprocessing.sequence import pad_sequences
  
  PAD_ID = 0
  _GO = "_GO"
  _END = "_END"
  _PAD = "_PAD"
  
  
  def create_voabulary(simple=None, word2vec_model_path='../zhihu-word2vec-title-desc.bin-100',
                       name_scope=''):  # zhihu-word2vec-multilabel.bin-100
      cache_path = '../cache_vocabulary_label_pik/' + name_scope + "_word_voabulary.pik"
      print("cache_path:", cache_path, "file_exists:", os.path.exists(cache_path))
      if os.path.exists(cache_path):  # 如果缓存文件存在，则直接读取
          with open(cache_path, 'r') as data_f:
              vocabulary_word2index, vocabulary_index2word = pickle.load(data_f)
              return vocabulary_word2index, vocabulary_index2word
      else:
          vocabulary_word2index = {}
          vocabulary_index2word = {}
          if simple is not None:
              word2vec_model_path = '../zhihu-word2vec.bin-100'
          print("create vocabulary. word2vec_model_path:", word2vec_model_path)
          model = word2vec.load(word2vec_model_path, kind='bin')
          vocabulary_word2index['PAD_ID'] = 0
          vocabulary_index2word[0] = 'PAD_ID'
          special_index = 0
          if 'biLstmTextRelation' in name_scope:
              vocabulary_word2index[
                  'EOS'] = 1  # a special token for biLstTextRelation model. which is used between two sentences.
              vocabulary_index2word[1] = 'EOS'
              special_index = 1
          for i, vocab in enumerate(model.vocab):
              vocabulary_word2index[vocab] = i + 1 + special_index
              vocabulary_index2word[i + 1 + special_index] = vocab
  
          # save to file system if vocabulary of words is not exists.
          if not os.path.exists(cache_path):  # 如果不存在写到缓存文件中
              with open(cache_path, 'a') as data_f:
                  pickle.dump((vocabulary_word2index, vocabulary_index2word), data_f)
      return vocabulary_word2index, vocabulary_index2word
  
  
  # create vocabulary of lables. label is sorted. 1 is high frequency, 2 is low frequency.
  def create_voabulary_label(voabulary_label='train-zhihu4-only-title-all.txt', name_scope='',
                             use_seq2seq=False):  # 'train-zhihu.txt'
      print("create_voabulary_label_sorted.started.traning_data_path:", voabulary_label)
      cache_path = '../cache_vocabulary_label_pik/' + name_scope + "_label_voabulary.pik"
      if os.path.exists(cache_path):  # 如果缓存文件存在，则直接读取
          with open(cache_path, 'r') as data_f:
              vocabulary_word2index_label, vocabulary_index2word_label = pickle.load(data_f)
              return vocabulary_word2index_label, vocabulary_index2word_label
      else:
          zhihu_f_train = codecs.open(voabulary_label, 'r', 'utf8')
          lines = zhihu_f_train.readlines()
          count = 0
          vocabulary_word2index_label = {}
          vocabulary_index2word_label = {}
          vocabulary_label_count_dict = {}  # {label:count}
          for i, line in enumerate(lines):
              if '__label__' in line:  # '__label__-2051131023989903826
                  label = line[line.index('__label__') + len('__label__'):].strip().replace("\n", "")
                  if vocabulary_label_count_dict.get(label, None) is not None:
                      vocabulary_label_count_dict[label] = vocabulary_label_count_dict[label] + 1
                  else:
                      vocabulary_label_count_dict[label] = 1
          list_label = sort_by_value(vocabulary_label_count_dict)
  
          print("length of list_label:", len(list_label));  # print(";list_label:",list_label)
          countt = 0
  
          ##########################################################################################
          if use_seq2seq:  # if used for seq2seq model,insert two special label(token):_GO AND _END
              i_list = [0, 1, 2];
              label_special_list = [_GO, _END, _PAD]
              for i, label in zip(i_list, label_special_list):
                  vocabulary_word2index_label[label] = i
                  vocabulary_index2word_label[i] = label
          #########################################################################################
          for i, label in enumerate(list_label):
              if i < 10:
                  count_value = vocabulary_label_count_dict[label]
                  print("label:", label, "count_value:", count_value)
                  countt = countt + count_value
              indexx = i + 3 if use_seq2seq else i
              vocabulary_word2index_label[label] = indexx
              vocabulary_index2word_label[indexx] = label
          print("count top10:", countt)
  
          # save to file system if vocabulary of words is not exists.
          if not os.path.exists(cache_path):  # 如果不存在写到缓存文件中
              with open(cache_path, 'a') as data_f:
                  pickle.dump((vocabulary_word2index_label, vocabulary_index2word_label), data_f)
      print("create_voabulary_label_sorted.ended.len of vocabulary_label:", len(vocabulary_index2word_label))
      return vocabulary_word2index_label, vocabulary_index2word_label
  
  
  def sort_by_value(d):
      items = d.items()
      backitems = [[v[1], v[0]] for v in items]
      backitems.sort(reverse=True)
      return [backitems[i][1] for i in range(0, len(backitems))]
  
  
  def create_voabulary_labelO():
      model = word2vec.load('zhihu-word2vec-multilabel.bin-100', kind='bin')  # zhihu-word2vec.bin-100
      count = 0
      vocabulary_word2index_label = {}
      vocabulary_index2word_label = {}
      label_unique = {}
      for i, vocab in enumerate(model.vocab):
          if '__label__' in vocab:  # '__label__-2051131023989903826
              label = vocab[vocab.index('__label__') + len('__label__'):]
              if label_unique.get(label, None) is None:  # 不曾出现过的话，保持到字典中
                  vocabulary_word2index_label[label] = count
                  vocabulary_index2word_label[count] = label  # ADD
                  count = count + 1
                  label_unique[label] = label
      return vocabulary_word2index_label, vocabulary_index2word_label
  
  
  def load_data_multilabel_new(vocabulary_word2index, vocabulary_word2index_label, valid_portion=0.05,
                               max_training_data=1000000,
                               traning_data_path='../train-zhihu4-only-title-all.txt', multi_label_flag=True,
                               use_seq2seq=False, seq2seq_label_length=6):  # n_words=100000,
      """
      input: a file path
      :return: train, test, valid. where train=(trainX, trainY). where
                  trainX: is a list of list.each list representation a sentence.trainY: is a list of label. each label is a number
      """
      # 1.load a zhihu data from file
      # example:"w305 w6651 w3974 w1005 w54 w109 w110 w3974 w29 w25 w1513 w3645 w6 w111 __label__-400525901828896492"
      print("load_data.started...")
      print("load_data_multilabel_new.training_data_path:", traning_data_path)
      zhihu_f = codecs.open(traning_data_path, 'r', 'utf8')  # -zhihu4-only-title.txt
      lines = zhihu_f.readlines()
      # 2.transform X as indices
      # 3.transform  y as scalar
      X = []
      Y = []
      Y_decoder_input = []  # ADD 2017-06-15
      for i, line in enumerate(lines):
          x, y = line.split('__label__')  # x='w17314 w5521 w7729 w767 w10147 w111'
          y = y.strip().replace('\n', '')
          x = x.strip()
          if i < 1:
              print(i, "x0:", x)  # get raw x
          # x_=process_one_sentence_to_get_ui_bi_tri_gram(x)
          x = x.split(" ")
          x = [vocabulary_word2index.get(e, 0) for e in
               x]  # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
          if i < 2:
              print(i, "x1:", x)  # word to index
          if use_seq2seq:  # 1)prepare label for seq2seq format(ADD _GO,_END,_PAD for seq2seq)
              ys = y.replace('\n', '').split(" ")  # ys is a list
              _PAD_INDEX = vocabulary_word2index_label[_PAD]
              ys_mulithot_list = [_PAD_INDEX] * seq2seq_label_length  # [3,2,11,14,1]
              ys_decoder_input = [_PAD_INDEX] * seq2seq_label_length
              # below is label.
              for j, y in enumerate(ys):
                  if j < seq2seq_label_length - 1:
                      ys_mulithot_list[j] = vocabulary_word2index_label[y]
              if len(ys) > seq2seq_label_length - 1:
                  ys_mulithot_list[seq2seq_label_length - 1] = vocabulary_word2index_label[_END]  # ADD END TOKEN
              else:
                  ys_mulithot_list[len(ys)] = vocabulary_word2index_label[_END]
  
              # below is input for decoder.
              ys_decoder_input[0] = vocabulary_word2index_label[_GO]
              for j, y in enumerate(ys):
                  if j < seq2seq_label_length - 1:
                      ys_decoder_input[j + 1] = vocabulary_word2index_label[y]
              if i < 10:
                  print(i, "ys:==========>0", ys)
                  print(i, "ys_mulithot_list:==============>1", ys_mulithot_list)
                  print(i, "ys_decoder_input:==============>2", ys_decoder_input)
          else:
              if multi_label_flag:  # 2)prepare multi-label format for classification
                  ys = y.replace('\n', '').split(" ")  # ys is a list
                  ys_index = []
                  for y in ys:
                      y_index = vocabulary_word2index_label[y]
                      ys_index.append(y_index)
                  ys_mulithot_list = transform_multilabel_as_multihot(ys_index)
              else:  # 3)prepare single label format for classification
                  ys_mulithot_list = vocabulary_word2index_label[y]
          if i <= 3:
              print("ys_index:")
              # print(ys_index)
              print(i, "y:", y, " ;ys_mulithot_list:", ys_mulithot_list)  # ," ;ys_decoder_input:",ys_decoder_input)
          X.append(x)
          Y.append(ys_mulithot_list)
          if use_seq2seq:
              Y_decoder_input.append(ys_decoder_input)  # decoder input
          # if i>50000:
          #    break
      # 4.split to train,test and valid data
      number_examples = len(X)
      print("number_examples:", number_examples)  #
      train = (X[0:int((1 - valid_portion) * number_examples)], Y[0:int((1 - valid_portion) * number_examples)])
      test = (X[int((1 - valid_portion) * number_examples) + 1:], Y[int((1 - valid_portion) * number_examples) + 1:])
      if use_seq2seq:
          train = train + (Y_decoder_input[0:int((1 - valid_portion) * number_examples)],)
          test = test + (Y_decoder_input[int((1 - valid_portion) * number_examples) + 1:],)
      # 5.return
      print("load_data.ended...")
      return train, test, test
  
  
  def load_data_multilabel_new_twoCNN(vocabulary_word2index, vocabulary_word2index_label, valid_portion=0.05,
                                      max_training_data=1000000,
                                      traning_data_path='train-zhihu4-only-title-all.txt',
                                      multi_label_flag=True):  # n_words=100000,
      """
      input: a file path
      :return: train, test, valid. where train=(trainX, trainY). where
                  trainX: is a list of list.each list representation a sentence.trainY: is a list of label. each label is a number
      """
      # 1.load a zhihu data from file
      # example:"w305 w6651 w3974 w1005 w54 w109 w110 w3974 w29 w25 w1513 w3645 w6 w111 __label__-400525901828896492"
      print("load_data.twoCNN.started...")
      print("load_data_multilabel_new_twoCNN.training_data_path:", traning_data_path)
      zhihu_f = codecs.open(traning_data_path, 'r', 'utf8')  # -zhihu4-only-title.txt
      lines = zhihu_f.readlines()
      # 2.transform X as indices
      # 3.transform  y as scalar
      X = []
      X2 = []
      Y = []
      count_error = 0
      for i, line in enumerate(lines):
          x, y = line.split('__label__')  # x='w17314 w5521 w7729 w767 w10147 w111'
          y = y.strip().replace('\n', '')
          x = x.strip()
          # print("x:===============>",x)
          try:
              x, x2 = x.split("\t")
          except Exception:
              print("x.split.error.", x, "count_error:", count_error)
              count_error += 1
              continue
          if i < 1:
              print(i, "x0:", x)  # get raw x
          # x_=process_one_sentence_to_get_ui_bi_tri_gram(x)
          x = x.split(" ")
          x = [vocabulary_word2index.get(e, 0) for e in
               x]  # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
          x2 = x2.split(" ")
          x2 = [vocabulary_word2index.get(e, 0) for e in x2]
          if i < 1:
              print(i, "x1:", x, "x2:", x2)  # word to index
          if multi_label_flag:
              ys = y.replace('\n', '').split(" ")  # ys is a list
              ys_index = []
              for y in ys:
                  y_index = vocabulary_word2index_label[y]
                  ys_index.append(y_index)
              ys_mulithot_list = transform_multilabel_as_multihot(ys_index)
          else:
              ys_mulithot_list = int(y)  # vocabulary_word2index_label[y]
          if i < 1:
              print(i, "y:", y, "ys_mulithot_list:", ys_mulithot_list)
          X.append(x)
          X2.append(x2)
          Y.append(ys_mulithot_list)
      # 4.split to train,test and valid data
      number_examples = len(X)
      print("number_examples:", number_examples)  #
      train = (X[0:int((1 - valid_portion) * number_examples)], X2[0:int((1 - valid_portion) * number_examples)],
               Y[0:int((1 - valid_portion) * number_examples)])
      test = (X[int((1 - valid_portion) * number_examples) + 1:], X2[int((1 - valid_portion) * number_examples) + 1:],
              Y[int((1 - valid_portion) * number_examples) + 1:])
      # 5.return
      print("load_data.ended...")
      return train, test, test
  
  
  def load_data(vocabulary_word2index, vocabulary_word2index_label, valid_portion=0.05, max_training_data=1000000,
                training_data_path='train-zhihu4-only-title-all.txt'):  # n_words=100000,
      """
      input: a file path
      :return: train, test, valid. where train=(trainX, trainY). where
                  trainX: is a list of list.each list representation a sentence.trainY: is a list of label. each label is a number
      """
      # 1.load a zhihu data from file
      # example:"w305 w6651 w3974 w1005 w54 w109 w110 w3974 w29 w25 w1513 w3645 w6 w111 __label__-400525901828896492"
      print("load_data.started...")
      zhihu_f = codecs.open(training_data_path, 'r', 'utf8')  # -zhihu4-only-title.txt
      lines = zhihu_f.readlines()
      # 2.transform X as indices
      # 3.transform  y as scalar
      X = []
      Y = []
      for i, line in enumerate(lines):
          x, y = line.split('__label__')  # x='w17314 w5521 w7729 w767 w10147 w111'
          y = y.replace('\n', '')
          x = x.replace("\t", ' EOS ').strip()
          if i < 5:
              print("x0:", x)  # get raw x
          # x_=process_one_sentence_to_get_ui_bi_tri_gram(x)
          # if i<5:
          #    print("x1:",x_) #
          x = x.split(" ")
          x = [vocabulary_word2index.get(e, 0) for e in
               x]  # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
          if i < 5:
              print("x1:", x)  # word to index
          y = vocabulary_word2index_label[y]  # np.abs(hash(y))
          X.append(x)
          Y.append(y)
      # 4.split to train,test and valid data
      number_examples = len(X)
      print("number_examples:", number_examples)  #
      train = (X[0:int((1 - valid_portion) * number_examples)], Y[0:int((1 - valid_portion) * number_examples)])
      test = (X[int((1 - valid_portion) * number_examples) + 1:], Y[int((1 - valid_portion) * number_examples) + 1:])
      # 5.return
      print("load_data.ended...")
      return train, test, test
  
  
  # 将一句话转化为(uigram,bigram,trigram)后的字符串
  def process_one_sentence_to_get_ui_bi_tri_gram(sentence, n_gram=3):
      """
      :param sentence: string. example:'w17314 w5521 w7729 w767 w10147 w111'
      :param n_gram:
      :return:string. example:'w17314 w17314w5521 w17314w5521w7729 w5521 w5521w7729 w5521w7729w767 w7729 w7729w767 w7729w767w10147 w767 w767w10147 w767w10147w111 w10147 w10147w111 w111'
      """
      result = []
      word_list = sentence.split(" ")  # [sentence[i] for i in range(len(sentence))]
      unigram = '';
      bigram = '';
      trigram = '';
      fourgram = ''
      length_sentence = len(word_list)
      for i, word in enumerate(word_list):
          unigram = word  # ui-gram
          word_i = unigram
          if n_gram >= 2 and i + 2 <= length_sentence:  # bi-gram
              bigram = "".join(word_list[i:i + 2])
              word_i = word_i + ' ' + bigram
          if n_gram >= 3 and i + 3 <= length_sentence:  # tri-gram
              trigram = "".join(word_list[i:i + 3])
              word_i = word_i + ' ' + trigram
          if n_gram >= 4 and i + 4 <= length_sentence:  # four-gram
              fourgram = "".join(word_list[i:i + 4])
              word_i = word_i + ' ' + fourgram
          if n_gram >= 5 and i + 5 <= length_sentence:  # five-gram
              fivegram = "".join(word_list[i:i + 5])
              word_i = word_i + ' ' + fivegram
          result.append(word_i)
      result = " ".join(result)
      return result
  
  
  # 加载数据，标签包含多个label：load data with multi-labels
  def load_data_with_multilabels(vocabulary_word2index, vocabulary_word2index_label, traning_path, valid_portion=0.05,
                                 max_training_data=1000000):  # n_words=100000,
      """
      input: a file path
      :return: train, test, valid. where train=(trainX, trainY). where
                  trainX: is a list of list.each list representation a sentence.trainY: is a list of label. each label is a number
      """
      # 1.load a zhihu data from file
      # example: 'w140 w13867 w10344 w2673 w9514 w269 w460 w6 w35053 w844 w10147 w111 __label__-2379261820462209275 -5535923551616745326 6038661761506862294'
      print("load_data_with_multilabels.ended...")
      zhihu_f = codecs.open(traning_path, 'r',
                            'utf8')  # ('/home/xul/xul/9_ZhihuCup/'+data_type+'-zhihu5-only-title-multilabel.txt', 'r', 'utf8') #home/xul/xul/9_ZhihuCup/'
      lines = zhihu_f.readlines()
      # 2.transform X as indices
      # 3.transform  y as scalar
      X = []
      Y = []
      Y_label1999 = []
      for i, line in enumerate(lines):
          # if i>max_training_data:
          #    break
          x, ys = line.split('__label__')  # x='w17314 w5521 w7729 w767 w10147 w111'
          ys = ys.replace('\n', '').split(" ")
          x = x.strip()
          if i < 5:
              print("x0:",
                    x)  # u'w4260 w4260w86860 w4260w86860w30907 w86860 w86860w30907 w86860w30907w11 w30907 w30907w11 w30907w11w31 w11 w11w31 w11w31w72 w31 w31w72 w31w72w166 w72 w72w166 w72w166w346 w166 w166w346 w166w346w2182 w346 w346w2182 w346w2182w224 w2182 w2182w224 w2182w224w2148 w224 w224w2148 w224w2148w6 w2148 w2148w6 w2148w6w2566 w6 w6w2566 w6w2566w25 w2566 w2566w25 w2566w25w1110 w25 w25w1110 w25w1110w111 w1110 w1110w111 w111'
          # x_=process_one_sentence_to_get_ui_bi_tri_gram(x)
          # if i < 5:
          #    print("x1:", x_)
          x = x.split(" ")
          x = [vocabulary_word2index.get(e, 0) for e in
               x]  # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
          if i < 5:
              print("x2:", x)
          # print("ys:",ys) #['501174938575526146', '-4317515119936650885']
          ys_list = []
          for y in ys:
              y_ = vocabulary_word2index_label[y]
              ys_list.append(y_)
          X.append(x)
          # TODO ys_list_array=transform_multilabel_as_multihot(ys_list) #it is 2-d array. [[ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.] [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]...]
          ys_list_ = proces_label_to_algin(ys_list)
          Y.append(ys_list_)
          # TODO Y_label1999.append(ys_list_array)
          if i == 0:
              print(X, Y)
              print(Y_label1999)
      # 4.split to train,test and valid data
      number_examples = len(X)
      train = (X[0:int((1 - valid_portion) * number_examples)], Y[0:int(
          (1 - valid_portion) * number_examples)])  # TODO Y_label1999[0:int((1 - valid_portion) * number_examples)]
      test = (X[int((1 - valid_portion) * number_examples) + 1:], Y[int((
                                                                                    1 - valid_portion) * number_examples) + 1:])  # TODO ,Y_label1999[int((1 - valid_portion) * number_examples) + 1:]
      print("load_data_with_multilabels.ended...")
      return train, test
  
  
  # 将LABEL转化为MULTI-HOT
  def transform_multilabel_as_multihot(label_list, label_size=1999):  # 1999label_list=[0,1,4,9,5]
      """
      :param label_list: e.g.[0,1,4]
      :param label_size: e.g.199
      :return:e.g.[1,1,0,1,0,0,........]
      """
      result = np.zeros(label_size)
      # set those location as 1, all else place as 0.
      result[label_list] = 1
      return result
  
  
  # 将LABEL转化为MULTI-HOT
  def transform_multilabel_as_multihotO(label_list, label_size=1999):  # 1999label_list=[0,1,4,9,5]
      batch_size = len(label_list)
      result = np.zeros((batch_size, label_size))
      # set those location as 1, all else place as 0.
      result[(range(batch_size), label_list)] = 1
      return result
  
  
  def load_final_test_data(file_path):
      final_test_file_predict_object = codecs.open(file_path, 'r', 'utf8')
      lines = final_test_file_predict_object.readlines()
      question_lists_result = []
      for i, line in enumerate(lines):
          question_id, question_string = line.split("\t")
          question_string = question_string.strip().replace("\n", "")
          question_lists_result.append((question_id, question_string))
      print("length of total question lists:", len(question_lists_result))
      return question_lists_result
  
  
  def load_data_predict(vocabulary_word2index, vocabulary_word2index_label, questionid_question_lists,
                        uni_to_tri_gram=False):  # n_words=100000,
      final_list = []
      for i, tuplee in enumerate(questionid_question_lists):
          queston_id, question_string_list = tuplee
          if uni_to_tri_gram:
              x_ = process_one_sentence_to_get_ui_bi_tri_gram(question_string_list)
              x = x_.split(" ")
          else:
              x = question_string_list.split(" ")
          x = [vocabulary_word2index.get(e, 0) for e in
               x]  # if can't find the word, set the index as '0'.(equal to PAD_ID = 0)
          if i <= 2:
              print("question_id:", queston_id);
              print("question_string_list:", question_string_list);
              print("x_indexed:", x)
          final_list.append((queston_id, x))
      number_examples = len(final_list)
      print("number_examples:", number_examples)  #
      return final_list
  
  
  def proces_label_to_algin(ys_list, require_size=5):
      """
      :param ys_list: a list
      :return: a list
      """
      ys_list_result = [0 for x in range(require_size)]
      if len(ys_list) >= require_size:  # 超长
          ys_list_result = ys_list[0:require_size]
      else:  # 太短
          if len(ys_list) == 1:
              ys_list_result = [ys_list[0] for x in range(require_size)]
          elif len(ys_list) == 2:
              ys_list_result = [ys_list[0], ys_list[0], ys_list[0], ys_list[1], ys_list[1]]
          elif len(ys_list) == 3:
              ys_list_result = [ys_list[0], ys_list[0], ys_list[1], ys_list[1], ys_list[2]]
          elif len(ys_list) == 4:
              ys_list_result = [ys_list[0], ys_list[0], ys_list[1], ys_list[2], ys_list[3]]
      return ys_list_result
  
  
  def write_uigram_to_trigram():
      pass
      # 1.read file.
      # 2.uigram--->trigram
      # 3.write each line to file system.
  
  
  def test_pad():
      trainX = 'w18476 w4454 w1674 w6 w25 w474 w1333 w1467 w863 w6 w4430 w11 w813 w4463 w863 w6 w4430 w111'
      trainX = trainX.split(" ")
      trainX = pad_sequences([[trainX]], maxlen=100, value=0.)
      print("trainX:", trainX)
  
  
  topic_info_file_path = 'topic_info.txt'
  
  
  def read_topic_info():
      f = codecs.open(topic_info_file_path, 'r', 'utf8')
      lines = f.readlines()
      dict_questionid_title = {}
      for i, line in enumerate(lines):
          topic_id, partent_ids, title_character, title_words, desc_character, decs_words = line.split("\t").strip()
          # print(i,"------------------------------------------------------")
          # print("topic_id:",topic_id)
          # print("partent_ids:",partent_ids)
          # print("title_character:",title_character)
          # print("title_words:",title_words)
          # print("desc_character:",desc_character)
          # print("decs_words:",decs_words)
          dict_questionid_title[topic_id] = title_words + " " + decs_words
      print("len(dict_questionid_title):", len(dict_questionid_title))
      return dict_questionid_title
  
  
  def stat_training_data_length():
      training_data = 'train-zhihu4-only-title-all.txt'
      f = codecs.open(training_data, 'r', 'utf8')
      lines = f.readlines()
      length_dict = {0: 0, 5: 0, 10: 0, 15: 0, 20: 0, 25: 0, 30: 0, 35: 0, 40: 0, 100: 0, 150: 0, 200: 0, 1500: 0}
      length_list = [0, 5, 10, 15, 20, 25, 30, 35, 40, 100, 150, 200, 1500]
      for i, line in enumerate(lines):
          line_list = line.split('__label__')[0].strip().split(" ")
          length = len(line_list)
          # print(i,"length:",length)
          for l in length_list:
              if length < l:
                  length = l
                  # print("length.assigned:",length)
                  break
          # print("length.before dict assign:", length)
          length_dict[length] = length_dict[length] + 1
      print("length_dict:", length_dict)
  
  
  if __name__ == '__main__':
      if __name__ == '__main__':
          if __name__ == '__main__':
              # 1.
              # vocabulary_word2index, vocabulary_index2word=create_voabulary()
              # vocabulary_word2index_label, vocabulary_index2word_label=create_voabulary_label()
              # load_data_with_multilabels(vocabulary_word2index,vocabulary_word2index_label,data_type='test')
              # 2.
              # sentence=u'我想开通创业板'
              # sentence='w18476 w4454 w1674 w6 w25 w474 w1333 w1467 w863 w6 w4430 w11 w813 w4463 w863 w6 w4430 w111'
              # result=process_one_sentence_to_get_ui_bi_tri_gram(sentence,n_gram=3)
              # print(len(result),"result:",result)
  
              # 3. transform to multilabel
              # label_list=[0,1,4,9,5]
              # result=transform_multilabel_as_multihot(label_list,label_size=15)
              # print("result:",result)
  
              # 4.load data for predict-----------------------------------------------------------------
              # file_path='test-zhihu-forpredict-v4only-title.txt'
              # questionid_question_lists=load_final_test_data(file_path)
  
              # vocabulary_word2index, vocabulary_index2word=create_voabulary()
              # vocabulary_word2index_label,_=create_voabulary_label()
              # final_list=load_data_predict(vocabulary_word2index, vocabulary_word2index_label, questionid_question_lists)
  
              # 5.process label require lengh
              # ys_list=[99999]
              # ys_list_result=proces_label_to_algin(ys_list,require_size=5)
              # print(ys_list,"ys_list_result1.:",ys_list_result)
              # ys_list=[99999,23423432,67566765]
              # ys_list_result=proces_label_to_algin(ys_list,require_size=5)
              # print(ys_list,"ys_list_result2.:",ys_list_result)
              # ys_list=[99999,23423432,67566765,23333333]
              # ys_list_result=proces_label_to_algin(ys_list,require_size=5)
              # print(ys_list,"ys_list_result2.:",ys_list_result)
              # ys_list = [99999, 23423432, 67566765,44543543,546546546,323423434]
              # ys_list_result = proces_label_to_algin(ys_list, require_size=5)
              # print(ys_list, "ys_list_result3.:", ys_list_result)
  
              # 6.create vocabulary label. sorted.
              # create_voabulary_label()
  
              # d={'a':3,'b':2,'c':11}
              # d_=sort_by_value(d)
              # print("d_",d_)
  
              # 7.
              # test_pad()
  
              # 8.read topic info
              # read_topic_info()
  
              # 9。
              stat_training_data_length()
  