code: def get_tokenizer(config):
      model_name = config["model_name"]
      if model_name in PLM_MODELS or model_name in RNN_MODELS:
          tokenizer_kwargs = config["tokenizer_kwargs"] or {}
          tokenizer_path = (
              config["tokenizer_path"] or config["model_path"]
          )
          if config["model_name"] in [
              "chinese-bart",
              "chinese-pegasus",
              "chinese-gpt2",
              "cpt",
          ]:
              tokenizer = BertTokenizer.from_pretrained(
                  tokenizer_path, **tokenizer_kwargs
              )
          else:
              tokenizer = AutoTokenizer.from_pretrained(
                  tokenizer_path, **tokenizer_kwargs
              )
  
          tokenizer.add_tokens(config["tokenizer_add_tokens"])
  
          # (1): tokenizer needs to add eos token
          if model_name in ["ctrl", "openai-gpt"]:
              tokenizer.add_special_tokens(({"eos_token": "</s>"}))
  
          # (2): tokenizer needs to add pad token
          if model_name in ["ctrl", "gpt2", "gpt_neo", "openai-gpt"]:
              tokenizer.pad_token = tokenizer.eos_token
  
          # (3): tokenizer needs to change replace eos token with sep token
          if model_name in ["cpm", "unilm", "xlm"]:
              tokenizer.eos_token = tokenizer.sep_token
  
          # (4): tokenizer needs to modify `build_inputs_with_special_tokens()` and `num_special_tokens_to_add()`
          if model_name in [
              "blenderbot-small",
              "cpm",
              "ctrl",
              "gpt2",
              "gpt_neo",
              "openai-gpt",
          ]:
              tokenizer.build_inputs_with_special_tokens = (
                  lambda t0, t1=None: t0 + [tokenizer.eos_token_id]
              )
              tokenizer.num_special_tokens_to_add = lambda: 1
          elif model_name in ["opt", "xlm-roberta"]:
              tokenizer.build_inputs_with_special_tokens = (
                  lambda t0, t1=None: [tokenizer.bos_token_id]
                  + t0
                  + [tokenizer.eos_token_id]
              )
              tokenizer.num_special_tokens_to_add = lambda: 2
  
          # (5): tokenizer needs to set src_lang, tgt_lang (used in translation task)
          if model_name in [
              "m2m_100",
              "mbart",
              "marian",
              "nllb",
              "xlm",
          ]:
              assert config["src_lang"] and config["tgt_lang"], (
                  model_name
                  + " needs to specify source language and target language "
                  "with `--src_lang=xx` and `--tgt_lang=xx`"
              )
              tokenizer.src_lang = config["src_lang"]
              tokenizer.tgt_lang = config["tgt_lang"]
  
      return tokenizer
  
imports: import os
  import datetime
  import importlib
  from accelerate.logging import get_logger
  from typing import Union, Optional
  import shutil
  import torch
  from accelerate.utils import set_seed
  from transformers import AutoTokenizer, BertTokenizer
  from .enum_type import PLM_MODELS, RNN_MODELS
label_desc: Use image node_id 11 for calling a global function with example usage: get_tokenizer(config) and returns: tokenizer