class_docstr: Megatron T5 pretraining
class_docstr_tok: ["Megatron","T5","pretraining"]
code: def add_special_tokens_to_tokenizer(
      cls,
      tokenizer,
      tokenizer_cfg,
      dataset_type="t5",
      add_sentinel_tokens_in_reverse_order=False,
      add_sentinel_tokens_first=False,
  ):
      # T5-related construction
      if (
          tokenizer_cfg.library == "huggingface"
          or tokenizer_cfg.library == "megatron"
      ):
          additional_tokens = {
              "additional_special_tokens": [
                  f"<extra_id_{i}>"
                  for i in range(
                      tokenizer_cfg.get("num_sentinel_tokens", 0)
                  )
              ]
          }
          if dataset_type == "ul2":
              mask_types = ["r", "s", "x"]
              for mask_type in mask_types:
                  additional_tokens["additional_special_tokens"].extend(
                      [f"<extra_id_{mask_type}>"]
                  )
          if additional_tokens["additional_special_tokens"]:
              tokenizer.add_special_tokens(additional_tokens)
  
      if tokenizer_cfg.library == "sentencepiece":
          # NOTE: This is an ugly way to support both NeMo-Megatron trained checkpoints and huggingface checkpoints.
          # Huggingface and Google checkpoints will add sentinel tokens first (right after the base vocabulary), but in NeMo-Megatron, we add <cls>, <sep>, <mask>, <pad>, <bos> etc. beofore sentinel tokens <extra_id_xx>.
          if add_sentinel_tokens_first:
              if tokenizer_cfg.get("num_sentinel_tokens", 0) > 0:
                  cls._add_sentinel_tokens(
                      tokenizer,
                      tokenizer_cfg.num_sentinel_tokens,
                      add_sentinel_tokens_in_reverse_order,
                  )
              cls._add_base_special_tokens(
                  tokenizer, is_huggingface_converted_model=True
              )
          else:
              cls._add_base_special_tokens(
                  tokenizer, is_huggingface_converted_model=False
              )
              if tokenizer_cfg.get("num_sentinel_tokens", 0) > 0:
                  cls._add_sentinel_tokens(
                      tokenizer,
                      tokenizer_cfg.num_sentinel_tokens,
                      add_sentinel_tokens_in_reverse_order,
                  )
  
          if dataset_type == "ul2":
              for mask_type in ["r", "s", "x"]:
                  if (
                      len(
                          tokenizer.text_to_ids(
                              f"▁<extra_id_{mask_type}>"
                          )
                      )
                      == 1
                  ):
                      tokenizer.special_token_to_id[
                          f"<extra_id_{mask_type}>"
                      ] = tokenizer.text_to_ids(
                          f"<extra_id_{mask_type}>"
                      )[
                          0
                      ]
                  else:
                      tokenizer.add_special_tokens(
                          [f"<extra_id_{mask_type}>"]
                      )
  
imports: import enum
  import math
  from omegaconf.dictconfig import DictConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from nemo.collections.nlp.data.language_modeling.megatron.dataset_utils import build_train_valid_test_datasets
  from nemo.collections.nlp.models.language_modeling.megatron_lm_encoder_decoder_model import MegatronLMEncoderDecoderModel
  from nemo.utils import logging
label_desc: Use image node_id 8 for calling the MegatronT5Model obj's underlying member method code with example usage: obj.add_special_tokens_to_tokenizer(cls, tokenizer, tokenizer_cfg, dataset_type, add_sentinel_tokens_in_reverse_order, add_sentinel_tokens_first) without return types