code: def check_tokenizer(
      model,
      tokenizer,
      model_name="unsloth/llama-2-7b-bnb-4bit",
      model_max_length=4096,
      padding_side="right",
      token=None,
      _reload=True,
  ):
      # Checks tokenizer for out of bounds ids.
      # Mainly a fix for https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha
      # where <sep> had token id=32002.
      # See https://huggingface.co/berkeley-nest/Starling-LM-7B-alpha/discussions/25
      # Seems like the Fast tokenizer in Rust breaks things!
  
      # We ignore some of them!
      if (
          tokenizer.__repr__().split("(", 1)[0]
          in IGNORED_TOKENIZER_CHECKING
      ):
          return tokenizer
      pass
  
      max_embedding_size = model.model.embed_tokens.weight.shape[0]
      added_tokens_fast = tokenizer.added_tokens_decoder
      added_tokens_fast = {
          index: str(value)
          for index, value in added_tokens_fast.items()
      }
      sorted_keys = sorted(added_tokens_fast)
      added_tokens_fast = {
          key: added_tokens_fast[key] for key in sorted_keys
      }
  
      for j, index in enumerate(added_tokens_fast.keys()):
          if index >= max_embedding_size:
              bad_indices = list(added_tokens_fast.keys())[j:]
              bad_tokens = list(added_tokens_fast.values())[j:]
  
              if not _reload:
                  # Try removing the token
                  added_tokens = [
                      str(x)
                      for x in tokenizer.added_tokens_decoder.values()
                  ]
                  special_tokens = tokenizer.special_tokens_map
                  import itertools
  
                  special_tokens = frozenset(
                      itertools.chain.from_iterable(
                          [x] if type(x) is str else x
                          for x in special_tokens.values()
                      )
                  )
                  can_be_removed1 = [
                      x for x in bad_tokens if x not in special_tokens
                  ]
                  can_be_removed2 = [
                      x
                      for x in can_be_removed1
                      if x in tokenizer._added_tokens_encoder.keys()
                  ]
  
                  # Check of extra tokens can in fact we removed!
  
                  if (len(can_be_removed1) == len(bad_tokens)) and (
                      len(can_be_removed2) == len(bad_tokens)
                  ):
                      # Yes it can be fixed!
                      for bad_token in can_be_removed1:
                          remove_id = tokenizer._added_tokens_encoder[
                              bad_token
                          ]
                          del tokenizer._added_tokens_decoder[remove_id]
                          del tokenizer._added_tokens_encoder[bad_token]
                      pass
                      # Confirm 1 more time!
                      if (
                          max(tokenizer.added_tokens_decoder.keys())
                          < max_embedding_size
                      ):
                          logger.warning_once(
                              f"Unsloth loaded a broken tokenizer `{model_name}`, but managed to repair it!\n"
                              f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"
                              "We removed these bad tokens. If you think this is incorrect, fix your tokenizer first."
                          )
                          return tokenizer
                      pass
                  pass
  
                  # :( Failure
                  raise RuntimeError(
                      f"Unsloth tried to load `{model_name}`, but cannot succeed.\n"
                      f"Tokens {bad_tokens} with ids {bad_indices} exceeds the max vocab size of {max_embedding_size}.\n"
                      f"Fix your tokenizer since it'll perform out of bounds memory accesses."
                  )
              pass
  
              # Try slow tokenizer which can fix things!
              tokenizer = AutoTokenizer.from_pretrained(
                  model_name,
                  model_max_length=model_max_length,
                  padding_side=padding_side,
                  token=token,
                  use_fast=False,
              )
              return check_tokenizer(
                  model=model,
                  tokenizer=tokenizer,
                  model_name=model_name,
                  model_max_length=model_max_length,
                  padding_side=padding_side,
                  token=token,
                  _reload=False,
              )
              break
          pass
      pass
      return tokenizer
  
file_path: unsloth/unsloth/models/_utils.py
imports: import torch
  from typing import Union, Optional, List, Any, Callable
  import numpy
  import warnings
  import gc
  import bitsandbytes
  from transformers.models.llama.modeling_llama import logger
  from transformers import AutoTokenizer
  from platform import system
  import xformers.ops.fmha
  from xformers import __version__
  from peft.tuners.lora.layer import LoraLayer
  from peft.tuners.lora import LoraLayer
label_desc: Use image node_id 3 for calling a global function with example usage: check_tokenizer(model, tokenizer, model_name, model_max_length, padding_side, token, _reload) and returns: tokenizer, tokenizer, check_tokenizer, tokenizer