code: def preprocess(cfg):
      """
      数据预处理阶段
      """
      logger.info("===== start preprocess data =====")
      train_fp = os.path.join(cfg.cwd, cfg.data_path, "train.csv")
      valid_fp = os.path.join(cfg.cwd, cfg.data_path, "valid.csv")
      test_fp = os.path.join(cfg.cwd, cfg.data_path, "test.csv")
      relation_fp = os.path.join(cfg.cwd, cfg.data_path, "relation.csv")
  
      logger.info("load raw files...")
      train_data = load_csv(train_fp)
      valid_data = load_csv(valid_fp)
      test_data = load_csv(test_fp)
      relation_data = load_csv(relation_fp)
  
      logger.info("clean data...")
      train_data = _clean_data(train_data)
      valid_data = _clean_data(valid_data)
      test_data = _clean_data(test_data)
  
      logger.info("convert relation into index...")
      rels = _handle_relation_data(relation_data)
      _add_relation_data(rels, train_data)
      _add_relation_data(rels, valid_data)
      _add_relation_data(rels, test_data)
  
      logger.info("verify whether use pretrained language models...")
      if cfg.model_name == "lm":
          logger.info(
              "use pretrained language models serialize sentence..."
          )
          _lm_serialize(train_data, cfg)
          _lm_serialize(valid_data, cfg)
          _lm_serialize(test_data, cfg)
      else:
          logger.info("serialize sentence into tokens...")
          serializer = Serializer(
              do_chinese_split=cfg.chinese_split, do_lower_case=True
          )
          serial = serializer.serialize
          _serialize_sentence(train_data, serial, cfg)
          _serialize_sentence(valid_data, serial, cfg)
          _serialize_sentence(test_data, serial, cfg)
  
          logger.info("build vocabulary...")
          vocab = Vocab("word")
          train_tokens = [d["tokens"] for d in train_data]
          valid_tokens = [d["tokens"] for d in valid_data]
          test_tokens = [d["tokens"] for d in test_data]
          sent_tokens = [*train_tokens, *valid_tokens, *test_tokens]
          for sent in sent_tokens:
              vocab.add_words(sent)
          vocab.trim(min_freq=cfg.min_freq)
  
          logger.info("convert tokens into index...")
          _convert_tokens_into_index(train_data, vocab)
          _convert_tokens_into_index(valid_data, vocab)
          _convert_tokens_into_index(test_data, vocab)
  
          logger.info("build position sequence...")
          _add_pos_seq(train_data, cfg)
          _add_pos_seq(valid_data, cfg)
          _add_pos_seq(test_data, cfg)
  
      logger.info("save data for backup...")
      os.makedirs(os.path.join(cfg.cwd, cfg.out_path), exist_ok=True)
      train_save_fp = os.path.join(cfg.cwd, cfg.out_path, "train.pkl")
      valid_save_fp = os.path.join(cfg.cwd, cfg.out_path, "valid.pkl")
      test_save_fp = os.path.join(cfg.cwd, cfg.out_path, "test.pkl")
      save_pkl(train_data, train_save_fp)
      save_pkl(valid_data, valid_save_fp)
      save_pkl(test_data, test_save_fp)
  
      if cfg.model_name != "lm":
          vocab_save_fp = os.path.join(
              cfg.cwd, cfg.out_path, "vocab.pkl"
          )
          vocab_txt = os.path.join(cfg.cwd, cfg.out_path, "vocab.txt")
          save_pkl(vocab, vocab_save_fp)
          logger.info("save vocab in txt file, for watching...")
          with open(vocab_txt, "w", encoding="utf-8") as f:
              f.write(os.linesep.join(vocab.word2idx.keys()))
  
      logger.info("===== end preprocess data =====")
  
file_path: DeepKE/src/deepke/relation_extraction/standard/tools/preprocess.py
imports: import os
  import logging
  from collections import OrderedDict
  from typing import List, Dict
  from transformers import BertTokenizer
  from .serializer import Serializer
  from .vocab import Vocab
  import sys
  from utils import save_pkl, load_csv
label_desc: Use image node_id 9 for calling a global function with example usage: preprocess(cfg) without return types