text: # encoding = utf-8
  
  """
  构建ner的数据集
  """
  
  import os
  import json
  from pip import main
  from tqdm import tqdm
  
  def generator_train_dataset():
      texts, labels, tags = [], [], set()
      with open("/home/macan/ml/workspace/PaddleNLP/examples/text_to_knowledge/wordtag/jieyu_parser.txt", "r", encoding="utf-8") as fd:
          for line in tqdm(fd):
              curr_text, curr_label = [], []
              line = line.strip()
              data = json.loads(line)
              text = data["text"]
              curr_text = list(text)
              items = data["items"]
              for item in items:
                  word = item["item"]
                  size = item["length"]
                  label = item["wordtag_label"]
                  # item_id = item["termid"]
                  tag = label + "-B"
                  curr_label.append(tag)
                  tags.add(tag)
                  for i in range(1, size, 1):
                      tag = label + "-I"
                      curr_label.append(tag)
                      tags.add(tag)
              # print(curr_label, curr_label, len(curr_label), len(curr_text))
              # 如果句子长度，超过102，对句子进行逗号的切分
              # if len(curr_text) > 102:
              #     text, label = [], []
              #     for char, la in zip(curr_text, curr_label):
              #         if char == "，" or char == "," and len(text) > 5:
              #             texts.append("\002".join(text))
              #             labels.append("\002".join(label))
              #             text, label = [], []
              #         else:
              #             text.append(char)
              #             label.append(la)
              #     if text and len(text) > 5:
              #         texts.append("\002".join(text))
              #         labels.append("\002".join(label))
              # else:
              #     texts.append("\002".join(curr_text))
              #     labels.append("\002".join(curr_label))
              
              if len(curr_text) != len(curr_label):
                  length = min(len(curr_text), len(curr_label))
                  texts.append("\002".join(curr_text[:length]))
                  labels.append("\002".join(curr_label[:length]))
              else:
                  texts.append("\002".join(curr_text))
                  labels.append("\002".join(curr_label))
              # if len(texts) > 10:
              #     break
      return texts, labels, tags
      
  def build_train_dataset():
      texts, labels, tags = generator_train_dataset()
  
      with open("./ner_train_dataset/dataset.txt", "w", encoding="utf-8") as fd:
          for text, label in tqdm(zip(texts, labels)):
              fd.write(text + "\t" + label + "\n")
              
      with open("./ner_train_dataset/tag.dic", "w", encoding="utf-8") as fd:
          for tag in tqdm(tags):
              fd.write(tag + "\n")
  
  
  if __name__ == '__main__':
      # build_train_dataset()
      
      
      chars = set()
      with open("/data/ml/data/xinhua_people_sentence.txt", "r", encoding="utf-8") as fd:
          for line in tqdm(fd):
              line = line.strip()
              [chars.add(c) for c in list(line)]
      
      with open("/home/macan/ml/workspace/PaddleNLP/examples/lexical_analysis/ner_train_dataset/word.dic", "w", encoding="utf-8") as fd:
          for line in chars:
              fd.write(line + "\n")
      #         curr_text, curr_label = [], []
      #         line = line.strip()
      #         data = json.loads(line)
      #         text = data["text"]
      #         # curr_text = list(text)
      #         lengths.append(len(text))
              
      # print(max(lengths), min(lengths), sum(lengths)/len(lengths))
                  
              
          