code: def _add_tags_to_tokens(self):
      mapping = self.mapping
      if (
          self.learn_weights
      ):  # add extra tokens to huggingface tokenizer
          self.mapping2id = {}
          self.mapping2targetid = {}
          for key, value in self.mapping.items():
              key_id = self.tokenizer.convert_tokens_to_ids(
                  self.tokenizer.tokenize(value[2:-2])
                  if "chinese" in self.bart_name
                  else self.tokenizer.tokenize(
                      value[2:-2], add_prefix_space=True
                  )
              )
              self.mapping2id[value] = key_id  # may be list
              self.mapping2targetid[key] = len(self.mapping2targetid)
      else:
          tokens_to_add = sorted(
              list(mapping.values()), key=lambda x: len(x), reverse=True
          )  #
          unique_no_split_tokens = (
              self.tokenizer.unique_no_split_tokens
          )  # no split
          sorted_add_tokens = sorted(
              list(tokens_to_add), key=lambda x: len(x), reverse=True
          )
          for tok in sorted_add_tokens:
              assert (
                  self.tokenizer.convert_tokens_to_ids([tok])[0]
                  == self.tokenizer.unk_token_id
              )  #
          self.tokenizer.unique_no_split_tokens = (
              unique_no_split_tokens + sorted_add_tokens
          )  # add to no_split_tokens
          self.tokenizer.add_tokens(sorted_add_tokens)
          self.mapping2id = {}  # tag to id
          self.mapping2targetid = {}  # tag to number
  
          for key, value in self.mapping.items():
              key_id = self.tokenizer.convert_tokens_to_ids(
                  self.tokenizer.tokenize(value)
              )
              assert len(key_id) == 1, value
              assert key_id[0] >= self.original_token_nums
              self.mapping2id[value] = key_id[0]  #
              self.mapping2targetid[key] = len(self.mapping2targetid)
  
file_path: DeepKE/src/deepke/name_entity_re/few_shot/module/datasets.py
imports: import torch
  from tqdm import tqdm
  import numpy
  from itertools import chain
  from torch.utils.data import Dataset, DataLoader
  from torch.nn.utils.rnn import pad_sequence
  from transformers import BartTokenizer, BertTokenizer
  import logging
label_desc: Use image node_id 4 for calling the ConllNERProcessor obj's underlying member method code with example usage: obj._add_tags_to_tokens() without return types