class_docstr: Megatron NMT training
class_docstr_tok: ["Megatron","NMT","training"]
code: def _build_tokenizer(self):
      # Instantiates tokenizers and register to be saved with NeMo Model archive
      # After this call, there will be self.encoder_tokenizer and self.decoder_tokenizer
      # Which can convert between tokens and token_ids for SRC and TGT languages correspondingly.
      encoder_tokenizer_model = self.register_artifact(
          "encoder_tokenizer.model",
          self._cfg.encoder_tokenizer.get("model"),
      )
      decoder_tokenizer_model = self.register_artifact(
          "decoder_tokenizer.model",
          self._cfg.decoder_tokenizer.get("model"),
      )
  
      (
          self.encoder_tokenizer,
          self.decoder_tokenizer,
      ) = MTEncDecModel.setup_enc_dec_tokenizers(
          encoder_tokenizer_library=self.encoder_tokenizer_library,
          encoder_tokenizer_model=encoder_tokenizer_model,
          encoder_bpe_dropout=self._cfg.encoder_tokenizer.get(
              "bpe_dropout", 0.0
          )
          if self._cfg.encoder_tokenizer.get("bpe_dropout", 0.0)
          is not None
          else 0.0,
          encoder_model_name=self._cfg.encoder_tokenizer.get(
              "type", None
          ),
          encoder_r2l=self._cfg.encoder_tokenizer.get("r2l", False),
          decoder_tokenizer_library=self.decoder_tokenizer_library,
          encoder_tokenizer_vocab_file=self._cfg.encoder_tokenizer.get(
              "vocab_file", None
          ),
          decoder_tokenizer_model=decoder_tokenizer_model,
          decoder_bpe_dropout=self._cfg.decoder_tokenizer.get(
              "bpe_dropout", 0.0
          )
          if self._cfg.decoder_tokenizer.get("bpe_dropout", 0.0)
          is not None
          else 0.0,
          decoder_model_name=self._cfg.encoder_tokenizer.get(
              "type", None
          ),
          decoder_r2l=self._cfg.decoder_tokenizer.get("r2l", False),
          encoder_sentencepiece_legacy=self._cfg.encoder_tokenizer.get(
              "sentencepiece_legacy", False
          ),
          decoder_sentencepiece_legacy=self._cfg.decoder_tokenizer.get(
              "sentencepiece_legacy", False
          ),
      )
  
imports: import enum
  import itertools
  import random
  from typing import List, Optional
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from omegaconf.listconfig import ListConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from sacrebleu import corpus_bleu
  from nemo.collections.nlp.data.common.sequence_to_sequence_dataset import BinarizedMemmapSequenceToSequenceDataset, TextMemmapSequenceToSequenceDataset
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers import MegatronPretrainingBatchSampler
  from nemo.collections.nlp.data.language_modeling.megatron.xlm_dataset import BinarizedMemmapCrossLingualMLMAndTranslationDataset, TextMemmapCrossLingualMLMAndTranslationDataset
  from nemo.collections.nlp.models.language_modeling.megatron_lm_encoder_decoder_model import MegatronLMEncoderDecoderModel
  from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model
  from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel
  from nemo.collections.nlp.modules.common.megatron.megatron_export import DecEmb, EncEmb, TokensHeadEmb
  from nemo.collections.nlp.modules.common.megatron.utils import get_iterator_k_split
  from nemo.collections.nlp.parts.nlp_overrides import GlobalBatchDataFetcher
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes import Exportable
  from nemo.utils import AppState, logging, timers
label_desc: Use image node_id 5 for calling the MegatronNMTModel obj's underlying member method code with example usage: obj._build_tokenizer() without return types