text: import json
  import logging
  import numpy as np
  import os
  import pickle
  from pyltp import Segmentor
  
  
  logger = logging.getLogger(__name__)
  logger.setLevel(logging.DEBUG)
  logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
  """
  SMP 分类工具类
  """
  
  
  
  class Data(object):
      """Load and parse the json data file
      
      Attributes:
          segment_data: 经过分词处理的query列表
          labels: 标签列表
      """
      
      LABELS = ['website', 'tvchannel', 'lottery', 'chat', 'match',
            'datetime', 'weather', 'bus', 'novel', 'video', 'riddle',
            'calc', 'telephone', 'health', 'contacts', 'epg', 'app', 'music',
            'cookbook', 'stock', 'map', 'message', 'poetry', 'cinemas', 'news',
            'flight', 'translation', 'train', 'schedule', 'radio', 'email']
  
  
      def __init__(self, data_file=None, ltp_data_path=None, max_length=None, stopwords=None):
          assert ltp_data_path is not None
          assert data_file is not None
   
          self.max_length = max_length
          self.data_file = data_file
          self.segmentor = Segmentor()
          
          self.segmentor.load(os.path.join(ltp_data_path, 'cws.model'))
          
          self.data = []
          self.padding_data = []
          self.__data__ = {}
          self.keys = []
          
          self.load_data(stopwords=stopwords)
          self.pad_data()
  
      def destory(self):
          self.segmentor.release()
          
      def get_metadata(self):
          """数据集的大小信息"""
          return len(self.data), self.max_length
      
      def load_data(self, stopwords=None):
          """load and parse the data"""
          with open(self.data_file, 'r', encoding='utf-8') as f:
              json_data = json.load(f)
              
          label_count = 0
          char_count = 0
          
          for key in json_data.keys():
              label = json_data.get(key).get('label')
              query = json_data.get(key).get('query')
  
              if label is None:
                  label = 'website'
                  
              self.__data__[key] = {
                  "query": query,
                  "label": label
              }
              self.keys.append(key)
              # TODO 截断 
              if self.max_length is not None:
                  words = list(self.segmentor.segment(query))[:self.max_length]
              else:
                  words = list(self.segmentor.segment(query))
              if stopwords is not None:
                  words = [word for word in words if word not in stopwords] 
              self.data.append((words, self.LABELS.index(label)))
          
          if self.max_length is None:
              self.max_length = max([len(x[0]) for x in self.data])
      
      def update_labels(self, labels):
          for index, key in enumerate(self.keys):
              self.__data__.get(key)['label'] = self.LABELS[int(labels[index])]
          return self
  
      def save_result(self, outputfile="out.json"):
          with open(outputfile, 'w', encoding='utf-8') as f:
              json.dump(self.__data__, f, ensure_ascii=False)
  
          
          
  
      def pad_data(self):
          max_length = self.max_length
          padding = [0] 
          for sentence, label in self.data:
              new_sentence = sentence[:max_length] if len(sentence) >= max_length else sentence + padding * (max_length - len(sentence))
              self.padding_data.append((np.array(new_sentence), label))
              
      
              
  
  def get_minibatches(data, minibatch_size, shuffle=True):
      """
      Iterates through the provided data one minibatch at at time. You can use this function to
      iterate through data in minibatches as follows:
  
          for inputs_minibatch in get_minibatches(inputs, minibatch_size):
              ...
  
      Or with multiple data sources:
  
          for inputs_minibatch, labels_minibatch in get_minibatches([inputs, labels], minibatch_size):
              ...
  
      Args:
          data: there are two possible values:
              - a list or numpy array
              - a list where each element is either a list or numpy array
          minibatch_size: the maximum number of items in a minibatch
          shuffle: whether to randomize the order of returned data
      Returns:
          minibatches: the return value depends on data:
              - If data is a list/array it yields the next minibatch of data.
              - If data a list of lists/arrays it returns the next minibatch of each element in the
                list. This can be used to iterate through multiple data sources
                (e.g., features and labels) at the same time.
  
      """
      list_data = type(data) is list and (type(data[0]) is list or type(data[0]) is np.ndarray)
      data_size = len(data[0]) if list_data else len(data)
      indices = np.arange(data_size)
      if shuffle:
          np.random.shuffle(indices)
      for minibatch_start in np.arange(0, data_size, minibatch_size):
          minibatch_indices = indices[minibatch_start:minibatch_start + minibatch_size]
          yield [minibatch(d, minibatch_indices) for d in data] if list_data \
              else minibatch(data, minibatch_indices)
  
  
  def minibatch(data, minibatch_idx):
      return data[minibatch_idx] if type(data) is np.ndarray else [data[i] for i in minibatch_idx]
  
  def minibatches(data, batch_size, shuffle=True):
      batches = [np.array(col) for col in zip(*data)]
      return get_minibatches(batches, batch_size, shuffle)
  
  
  def load_word_embedding(input_file='./data/sgns.target.word-word.dynwin5.thr10.neg5.dim300.iter5', cache=None):
      assert input_file is not None
      if cache is not None and os.path.exists(cache):
          with open(cache, 'rb') as f:
              embedding = pickle.load(f)
              token2id = pickle.load(f)
          return embedding, token2id
      with open(input_file, encoding='utf-8', errors='ignore') as f:
          metadata = next(f).strip()
          word_sum, dim = metadata.split(' ')
          embedding = np.array(np.random.randn(int(word_sum) + 1, int(dim)), dtype=np.float32)
          token2id = {}
          count = 1
          
          for line in f:
              items = line.strip().split(' ')
              token2id[items[0]] = count
              embedding[count] = np.array(items[1:])
              count += 1
          logging.info("Initialized embedding.")
          if cache is not None:
              with open("cache", 'wb') as f:
                  pickle.dump(embedding, f)
                  pickle.dump(token2id, f)
          return embedding, token2id
  
  def load_stopwords(input_file='data/stopwords.txt'):
      """some doces"""
      with open(input_file, encoding='utf-8', errors='ignore') as f:
          stopwords = set()
          for line in f:
              stopwords.add(line.strip())
      return stopwords
  
  def embedding_simplify(embedding, token2id, data):
      """保留需用到的词向量, 全部太大了"""
      new_embedding = []
      for item in data:
          for word in item[0]:
              index = token2id.get(word)
              if index is not None:
                  new_embedding.append((word, embedding[index]))
      with open('embeding_terse', 'w', encoding='utf-8') as f:
          f.write("%d %d\n" % (len(new_embedding), 300))
          for item in new_embedding:
              f.write("%s %s\n" % (item[0], " ".join([str(x) for x in item[1]])))
  
  
  def env_testing(_):
      """测试程序执行的环境"""
  
      path = "./" #文件夹目录
      files= os.listdir(path) #得到文件夹下的所有文件名称
      s = []
      for _file in files: #遍历文件夹
          print(_file)
  
  
  
  def test1():
      train_data = Data('./data/train.json', 'L:\\workspace\\ltp_data\\')
      
      data = train_data.padding_data
      batch = minibatches(data, 12)
      for x in batch: 
          print(x)
  
  def diff(grand_file, predict_file, outputfile):
      with open(grand_file, 'r', encoding='utf-8') as f:
          json_data1 = json.load(f)
      with open(predict_file, 'r', encoding='utf-8') as f:
          json_data2 = json.load(f)
      
      result = {}
  
      
      for key in json_data1.keys():
          label1 = json_data1.get(key).get('label')
          label2 = json_data2.get(key).get('label')
          if label1 != label2:
              result[key] = {}
              result[key]['query'] = json_data1.get(key).get('query')
              result[key]['grand'] = label1
              result[key]['predict'] = label2
  
      with open(outputfile, 'w', encoding='utf-8') as f:
          json.dump(result, f, ensure_ascii=False)
  
          
  
  if __name__ == '__main__':
  
      grand_file = 'data/dev.json'
      predict_file = 'out.json'
      diff(grand_file, predict_file, 'diff.json')
      exit()
  
      pass
      stopwords = load_stopwords()
      train_data = Data('./data/train.json', 'L:\\workspace\\ltp_data\\', stopwords=stopwords)
      prediction = []
      for i in range(len(train_data.data)):
          prediction.append(0)
      
      train_data.update_labels(prediction)
      pass
          
      # data = train_data.data
      # pass
      # embedding, token2id = load_word_embedding(cache='cache')
      # embedding_simplify(embedding, token2id, data)
      
  
  