code: def get_tokenizer(cfg: Any):
      kwargs = dict(
          revision=cfg.environment.huggingface_branch,
          add_prefix_space=cfg.tokenizer.add_prefix_space,
          use_fast=cfg.tokenizer.use_fast,
          trust_remote_code=cfg.environment.trust_remote_code,
          use_auth_token=os.getenv("HUGGINGFACE_TOKEN"),
      )
  
      try:
          tokenizer = AutoTokenizer.from_pretrained(
              cfg.llm_backbone, **kwargs
          )
      except TypeError:
          # TypeError: RWForCausalLM.__init__() got
          # an unexpected keyword argument 'use_auth_token'
          kwargs.pop("use_auth_token")
          tokenizer = AutoTokenizer.from_pretrained(
              cfg.llm_backbone, **kwargs
          )
      tokenizer.padding_side = getattr(
          cfg.tokenizer, "_padding_side", tokenizer.padding_side
      )
  
      # if the eos token is an empty string, we assign it to a token
      if tokenizer.eos_token == "":
          tokenizer.add_special_tokens({"eos_token": "</s>"})
          tokenizer.eos_token = "</s>"
  
      if tokenizer.pad_token is None:
          if tokenizer.unk_token is not None:
              tokenizer.pad_token = tokenizer.unk_token
          else:
              tokenizer.pad_token = tokenizer.eos_token
      if tokenizer.bos_token is None:
          tokenizer.bos_token = tokenizer.eos_token
      if tokenizer.cls_token is None:
          tokenizer.cls_token = tokenizer.eos_token
      if tokenizer.sep_token is None:
          tokenizer.sep_token = tokenizer.eos_token
  
      cfg._tokenizer_sep_token = tokenizer.sep_token
  
      if tokenizer.unk_token_id is not None:
          cfg._tokenizer_mask_token_id = tokenizer.unk_token_id
      elif tokenizer.mask_token_id is not None:
          cfg._tokenizer_mask_token_id = tokenizer.mask_token_id
      elif tokenizer.pad_token_id is not None:
          cfg._tokenizer_mask_token_id = tokenizer.pad_token_id
      else:
          # setting the mask token id to the last token in the vocabulary
          # this usually is a safe choice and mostly refers to eos token
          cfg._tokenizer_mask_token_id = len(tokenizer) - 1
  
      cfg._tokenizer_eos_token = tokenizer.eos_token
  
      if hasattr(cfg.prediction, "stop_tokens"):
          set_stop_token_ids(cfg, tokenizer)
      cfg.tokenizer._vocab_length = len(tokenizer)
  
      return tokenizer
  
imports: import codecs
  import logging
  import os
  from typing import Any
  from transformers import AutoTokenizer
label_desc: Use image node_id 2 for calling a global function with example usage: get_tokenizer(cfg) and returns: tokenizer