class_docstr: Megatron NMT training
class_docstr_tok: ["Megatron","NMT","training"]
code: def _build_vocab(self):
      if (
          hasattr(self.cfg, "data")
          and self.cfg.train_ds.get("objective", "nmt") != "nmt-xlm"
      ):
          if hasattr(self.cfg.data, "dataset_type"):
              # This happens only when restoring a pre-trained model. We need to add all of the special tokens that were added while pre-training to avoid a checkpoint shape mismatch while restoring.
              MegatronT5Model.add_special_tokens_to_tokenizer(
                  tokenizer=self.encoder_tokenizer,
                  tokenizer_cfg=self.cfg.encoder_tokenizer,
                  dataset_type=self.cfg.data.dataset_type,
              )
              MegatronT5Model.add_special_tokens_to_tokenizer(
                  tokenizer=self.decoder_tokenizer,
                  tokenizer_cfg=self.cfg.decoder_tokenizer,
                  dataset_type=self.cfg.data.dataset_type,
              )
  
      if self.cfg.train_ds.get("objective", "nmt") == "nmt-xlm":
          if self.cfg.encoder_tokenizer.library != "sentencepiece":
              raise ValueError(
                  f"NMT-XLM objective requires sentencepiece tokenizer, but got encoder tokenizer library : {self.cfg.encoder_tokenizer.library}"
              )
          if self.cfg.decoder_tokenizer.library != "sentencepiece":
              raise ValueError(
                  f"NMT-XLM objective requires sentencepiece tokenizer, but got decoder tokenizer library : {self.cfg.decoder_tokenizer.library}"
              )
          MegatronT5Model.add_special_tokens_to_tokenizer(
              tokenizer=self.encoder_tokenizer,
              tokenizer_cfg=self.cfg.encoder_tokenizer,
              dataset_type="ul2",
          )
          MegatronT5Model.add_special_tokens_to_tokenizer(
              tokenizer=self.decoder_tokenizer,
              tokenizer_cfg=self.cfg.decoder_tokenizer,
              dataset_type="ul2",
          )
  
      # Set up pre and post processors as well.
      # NOTE: multilingual language tokens are set up after other special tokens such as eos, pad, sentinel tokens etc are added.
      if self.multilingual:
          (
              self.source_processor_list,
              self.target_processor_list,
              self.multilingual_lang_to_id,
          ) = MTEncDecModel.setup_multilingual_ids_and_processors(
              src_language=self.src_language,
              tgt_language=self.tgt_language,
              encoder_tokenizer=self.encoder_tokenizer,  # Multilingual training requires shared tokenizers.
              decoder_tokenizer=self.decoder_tokenizer,
              encoder_tokenizer_library=self.encoder_tokenizer_library,
              decoder_tokenizer_library=self.decoder_tokenizer_library,
          )
          self.multilingual_ids = list(
              self.multilingual_lang_to_id.values()
          )
      else:
          # After this call, the model will have  self.source_processor and self.target_processor objects
          (
              self.source_processor,
              self.target_processor,
          ) = MTEncDecModel.setup_pre_and_post_processing_utils(
              self.src_language,
              self.tgt_language,
              self.encoder_tokenizer_library,
              self.decoder_tokenizer_library,
          )
          self.multilingual_ids = [None]
  
      self.padded_vocab_size = self._vocab_size_with_padding(
          orig_vocab_size=self.encoder_tokenizer.vocab_size,
          make_vocab_size_divisible_by=self._cfg.get(
              "make_vocab_size_divisible_by", 128
          ),
          tensor_model_parallel_size=self._cfg.get(
              "tensor_model_parallel_size", 1
          ),
      )
  
imports: import enum
  import itertools
  import random
  from typing import List, Optional
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from omegaconf.listconfig import ListConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from sacrebleu import corpus_bleu
  from nemo.collections.nlp.data.common.sequence_to_sequence_dataset import BinarizedMemmapSequenceToSequenceDataset, TextMemmapSequenceToSequenceDataset
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers import MegatronPretrainingBatchSampler
  from nemo.collections.nlp.data.language_modeling.megatron.xlm_dataset import BinarizedMemmapCrossLingualMLMAndTranslationDataset, TextMemmapCrossLingualMLMAndTranslationDataset
  from nemo.collections.nlp.models.language_modeling.megatron_lm_encoder_decoder_model import MegatronLMEncoderDecoderModel
  from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model
  from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel
  from nemo.collections.nlp.modules.common.megatron.megatron_export import DecEmb, EncEmb, TokensHeadEmb
  from nemo.collections.nlp.modules.common.megatron.utils import get_iterator_k_split
  from nemo.collections.nlp.parts.nlp_overrides import GlobalBatchDataFetcher
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes import Exportable
  from nemo.utils import AppState, logging, timers
label_desc: Use image node_id 6 for calling the MegatronNMTModel obj's underlying member method code with example usage: obj._build_vocab() without return types