code: def load_tokenizer(cfg):
      model_config = load_model_config(cfg)
      tokenizer_kwargs = {}
      use_fast = True  # this is the default
  
      if cfg.tokenizer_use_fast is not None:
          use_fast = cfg.tokenizer_use_fast
      if cfg.tokenizer_legacy is not None:
          # True is the default w/ https://github.com/huggingface/transformers/pull/25224
          tokenizer_kwargs["legacy"] = cfg.tokenizer_legacy
  
      tokenizer_cls = AutoTokenizer
      if cfg.tokenizer_type:
          tokenizer_cls = getattr(transformers, cfg.tokenizer_type)
  
      tokenizer_config = (
          cfg.tokenizer_config
          or cfg.base_model_config
          or cfg.base_model
      )
      tokenizer = tokenizer_cls.from_pretrained(
          tokenizer_config,
          trust_remote_code=cfg.trust_remote_code or False,
          use_fast=use_fast,
          **tokenizer_kwargs,
      )
  
      if (
          tokenizer.__class__.__name__
          in [
              "LlamaTokenizer",
              "LlamaTokenizerFast",
              "CodeLlamaTokenizer",
              "CodeLlamaTokenizerFast",
          ]
          and hasattr(tokenizer, "pad_token")
          and not tokenizer.pad_token
      ):
          # set a pad_token, but use eos_token so we don't add a new token
          tokenizer.pad_token = LLAMA_DEFAULT_EOS_TOKEN
  
      if tokenizer.__class__.__name__ == "GPTNeoXTokenizerFast":
          tokenizer.add_special_tokens({"pad_token": "[PAD]"})
          os.environ["TOKENIZERS_PARALLELISM"] = "false"
  
      # Mistral's official FA implementation requires left padding
      if (
          cfg.is_mistral_derived_model
          and cfg.flash_attention
          and not cfg.sample_packing
      ):
          tokenizer.padding_side = "left"
  
      # Qwen base only has single token, so we need to set the special tokens
      if cfg.is_qwen_derived_model:
          token_ids = [
              "bos_token_id",
              "eos_token_id",
              "pad_token_id",
              "unk_token_id",
          ]
          for attr_name in token_ids:
              if getattr(tokenizer, attr_name) is None:
                  setattr(tokenizer, attr_name, tokenizer.eod_id)
  
          token_names = [
              "bos_token",
              "eos_token",
              "pad_token",
              "unk_token",
          ]
          for attr_name in token_names:
              if getattr(tokenizer, attr_name) is None:
                  setattr(tokenizer, attr_name, "<|endoftext|>")
  
      if cfg.special_tokens:
          lora_modules_to_save = get_linear_embedding_layers(
              model_config.model_type
          )
          for k, val in cfg.special_tokens.items():
              # check if new special token is not already in tokenizer and
              # is adapter training to make sure lora_modules_to_save is set
              # pylint: disable=too-many-boolean-expressions
              if (
                  (
                      getattr(tokenizer, k) is None
                      or getattr(tokenizer, k) != val
                  )
                  and cfg.adapter
                  and (
                      not cfg.lora_modules_to_save
                      or not all(
                          x in cfg.lora_modules_to_save
                          for x in lora_modules_to_save
                      )
                  )
              ):
                  lora_modules_to_save = ", ".join(
                      [f"`{x}`" for x in lora_modules_to_save]
                  )
                  raise ValueError(
                      f"Please set lora_modules_to_save to {lora_modules_to_save} when using an adapter and changing the special tokens."
                  )
  
              tokenizer.add_special_tokens(
                  {
                      k: AddedToken(
                          val,
                          rstrip=False,
                          lstrip=False,
                          normalized=False,
                      )
                  }
              )
  
          # If we add bos_token and eos_token, we need to update the post processor to
          # handle them correctly.
          # https://github.com/huggingface/transformers/pull/24132
          bos_or_eos_in_special_tokens = (
              "bos_token" in cfg.special_tokens
              and "eos_token" in cfg.special_tokens
          )
          if (
              tokenizer.__class__.__name__
              in (
                  "LlamaTokenizerFast",
                  "CodeLlamaTokenizerFast",
              )
              and bos_or_eos_in_special_tokens
          ):
              tokenizer.update_post_processor()
  
      if cfg.tokens:
          tokenizer.add_tokens(
              [
                  AddedToken(
                      token,
                      rstrip=False,
                      lstrip=False,
                      normalized=False,
                  )
                  for token in cfg.tokens
              ]
          )
  
      LOG.debug(
          f"EOS: {tokenizer.eos_token_id} / {tokenizer.eos_token}"
      )
      LOG.debug(
          f"BOS: {tokenizer.bos_token_id} / {tokenizer.bos_token}"
      )
      LOG.debug(
          f"PAD: {tokenizer.pad_token_id} / {tokenizer.pad_token}"
      )
      LOG.debug(
          f"UNK: {tokenizer.unk_token_id} / {tokenizer.unk_token}"
      )
  
      if cfg.chat_template:
          tokenizer.chat_template = chat_templates(cfg.chat_template)
      else:
          LOG.info(
              "No Chat template selected. Consider adding a chat template for easier inference."
          )
      return tokenizer
  
imports: import logging
  import math
  import os
  from typing import Any, Optional, Tuple, Union
  import addict
  import bitsandbytes
  import torch
  import transformers
  from optimum.bettertransformer import BetterTransformer
  from peft import PeftConfig, prepare_model_for_kbit_training
  from peft.tuners.lora import QuantLinear
  from transformers import AddedToken, AutoConfig, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, GPTQConfig, PreTrainedModel, PreTrainedTokenizerBase
  from transformers.deepspeed import is_deepspeed_zero3_enabled
  from axolotl.models.mamba import fix_mamba_attn_for_loss
  from axolotl.prompt_tokenizers import LLAMA_DEFAULT_EOS_TOKEN
  from axolotl.utils.bench import log_gpu_memory_usage
  from axolotl.utils.chat_templates import chat_templates
  from axolotl.utils.dict import DictDefault
  from axolotl.utils.lora_embeddings import get_linear_embedding_layers
label_desc: Use image node_id 3 for calling a global function with example usage: load_tokenizer(cfg) and returns: tokenizer