code: def process(self, data_dict):
      target_shift = len(self.mapping) + 2
  
      def prepare_target(item):
          raw_word = item["raw_word"]
          word_bpes = [
              [
                  self.tokenizer.cls_token_id
                  if "chinese" in self.bart_name
                  else self.tokenizer.bos_token_id
              ]
          ]
          first = []
          cur_bpe_len = 1
          for word in raw_word:
              bpes = (
                  self.tokenizer.tokenize(word)
                  if "chinese" in self.bart_name
                  else self.tokenizer.tokenize(
                      word, add_prefix_space=True
                  )
              )
              bpes = self.tokenizer.convert_tokens_to_ids(bpes)
              first.append(cur_bpe_len)
              cur_bpe_len += len(bpes)
              word_bpes.append(bpes)
          assert first[-1] + len(bpes) == sum(map(len, word_bpes))
          word_bpes.append(
              [
                  self.tokenizer.sep_token_id
                  if "chinese" in self.bart_name
                  else self.tokenizer.eos_token_id
              ]
          )
          assert len(first) == len(raw_word) == len(word_bpes) - 2
  
          lens = list(map(len, word_bpes))
          cum_lens = np.cumsum(lens).tolist()
  
          entity_spans = item["entity_span"]  # [(s1, e1, s2, e2), ()]
          entity_tags = item["entity_tag"]  # [tag1, tag2...]
          entities = item["entity"]  # [[ent1, ent2,], [ent1, ent2]]
          target = [0]
          pairs = []
  
          first = list(range(cum_lens[-1]))
  
          assert len(entity_spans) == len(entity_tags)  #
          for idx, (entity, tag) in enumerate(
              zip(entity_spans, entity_tags)
          ):
              cur_pair = []
              num_ent = len(entity) // 2
              for i in range(num_ent):
                  start = entity[2 * i]
                  end = entity[2 * i + 1]
                  cur_pair_ = []
                  cur_pair_.extend(
                      [cum_lens[k] for k in list(range(start, end))]
                  )
                  cur_pair.extend([p + target_shift for p in cur_pair_])
              for _, (j, word_idx) in enumerate(
                  zip((cur_pair[0], cur_pair[-1]), (0, -1))
              ):
                  j = j - target_shift
              assert all(
                  [
                      cur_pair[i] < cum_lens[-1] + target_shift
                      for i in range(len(cur_pair))
                  ]
              )
  
              cur_pair.append(self.mapping2targetid[tag] + 2)
              pairs.append([p for p in cur_pair])
          target.extend(list(chain(*pairs)))
          target.append(1)
  
          word_bpes = list(chain(*word_bpes))
          assert len(word_bpes) < 500
  
          dict = {
              "tgt_tokens": target,
              "target_span": pairs,
              "src_tokens": word_bpes,
              "first": first,
              "src_seq_len": len(word_bpes),
              "tgt_seq_len": len(target),
          }
          return dict
  
      logger.info("Process data...")
      for raw_word, raw_target, entity, entity_tag, entity_span in tqdm(
          zip(
              data_dict["raw_words"],
              data_dict["raw_targets"],
              data_dict["entities"],
              data_dict["entity_tags"],
              data_dict["entity_spans"],
          ),
          total=len(data_dict["raw_words"]),
          desc="Processing",
      ):
          item_dict = prepare_target(
              {
                  "raw_word": raw_word,
                  "raw_target": raw_target,
                  "entity": entity,
                  "entity_tag": entity_tag,
                  "entity_span": entity_span,
              }
          )
          # add item_dict to data_dict
          for key, value in item_dict.items():
              if key in data_dict:
                  data_dict[key].append(value)
              else:
                  data_dict[key] = [value]
      return data_dict
  
file_path: DeepKE/src/deepke/name_entity_re/few_shot/module/datasets.py
imports: import torch
  from tqdm import tqdm
  import numpy
  from itertools import chain
  from torch.utils.data import Dataset, DataLoader
  from torch.nn.utils.rnn import pad_sequence
  from transformers import BartTokenizer, BertTokenizer
  import logging
label_desc: Use image node_id 3 for calling the ConllNERProcessor obj's underlying member method code with example usage: obj.process(data_dict) and returns: data_dict, dict