class_docstr: Base class for NLP Models.
      
class_docstr_tok: ["Base","class","for","NLP","Models","."]
docstr: Instantiates tokenizer based on config and registers tokenizer artifacts.
  
     If model is being restored from .nemo file then the tokenizer.vocab_file will
     be used (if it exists).
  
     Otherwise, we will use the vocab file provided in the config (if it exists).
  
     Finally, if no vocab file is given (this happens frequently when using HF),
     we will attempt to extract the vocab from the tokenizer object and then register it.
  
  Args:
      cfg (DictConfig): Tokenizer config
docstr_tok: ["Instantiates","tokenizer","based","on","config","and","registers","tokenizer","artifacts",".","If","model","is","being","restored","from",".nemo","file","then","the","tokenizer.vocab_file","will","be","used","(","if","it","exists",")",".","Otherwise",",","we","will","use","the","vocab","file","provided","in","the","config","(","if","it","exists",")",".","Finally",",","if","no","vocab","file","is","given","(","this","happens","frequently","when","using","HF",")",",","we","will","attempt","to","extract","the","vocab","from","the","tokenizer","object","and","then","register","it",".","Args",":","cfg","(","DictConfig",")",":","Tokenizer","config"]
code: def setup_tokenizer(self, cfg: DictConfig):
      """Instantiates tokenizer based on config and registers tokenizer artifacts.
  
         If model is being restored from .nemo file then the tokenizer.vocab_file will
         be used (if it exists).
  
         Otherwise, we will use the vocab file provided in the config (if it exists).
  
         Finally, if no vocab file is given (this happens frequently when using HF),
         we will attempt to extract the vocab from the tokenizer object and then register it.
  
      Args:
          cfg (DictConfig): Tokenizer config
      """
      vocab_file = None
      if cfg.get("vocab_file"):
          vocab_file = self.register_artifact(
              config_path="tokenizer.vocab_file", src=cfg.vocab_file
          )
      # only load tokenizer if vocab_file and tokenizer_model is not None
      if cfg.tokenizer_name or vocab_file or cfg.tokenizer_model:
          self.tokenizer = get_tokenizer(
              tokenizer_name=cfg.tokenizer_name,
              vocab_file=vocab_file,
              special_tokens=OmegaConf.to_container(cfg.special_tokens)
              if cfg.special_tokens
              else None,
              tokenizer_model=self.register_artifact(
                  config_path="tokenizer.tokenizer_model",
                  src=cfg.tokenizer_model,
              ),
          )
  
      if vocab_file is None:
          # when there is no vocab file we try to get the vocab from the tokenizer and register it
          self._register_vocab_from_tokenizer(
              vocab_file_config_path="tokenizer.vocab_file", cfg=cfg
          )
  
imports: import copy
  import hashlib
  import json
  import os
  from typing import Any, Mapping, Optional, Union
  import torch
  from lightning_fabric.utilities.cloud_io import _load
  from omegaconf import DictConfig, OmegaConf
  from pytorch_lightning import Trainer
  from pytorch_lightning.core.saving import _load_state
  from pytorch_lightning.core.saving import load_hparams_from_tags_csv, load_hparams_from_yaml
  from pytorch_lightning.utilities import rank_zero_only
  from pytorch_lightning.utilities.migration import pl_legacy_patch
  from transformers import TRANSFORMERS_CACHE
  from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer
  from nemo.collections.nlp.modules import BertModule
  from nemo.collections.nlp.modules.common.huggingface.huggingface_utils import VOCAB_FILE_NAME
  from nemo.collections.nlp.modules.common.lm_utils import get_lm_model
  from nemo.collections.nlp.modules.common.megatron.megatron_utils import MEGATRON_CONFIG_MAP, get_megatron_pretrained_bert_models
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
  from nemo.collections.nlp.parts.nlp_overrides import NLPSaveRestoreConnector
  from nemo.core.classes import ModelPT
  from nemo.core.classes.exportable import Exportable
  from nemo.core.connectors.save_restore_connector import SaveRestoreConnector
  from nemo.utils import AppState, logging
label_desc: Use image node_id 4 for calling the NLPModel obj's underlying member method code with example usage: obj.setup_tokenizer(cfg) without return types