class_docstr: Megatron NMT training
class_docstr_tok: ["Megatron","NMT","training"]
code: def _instantiate_memmap_dataset(
      self,
      cfg,
      src_file,
      tgt_file,
      src_language,
      tgt_language,
      num_samples,
      prepend_id=None,
  ):
      if cfg.dataset_type == "bin_memmap":
          if cfg.get("objective", "nmt") == "nmt":
              dataset = BinarizedMemmapSequenceToSequenceDataset(
                  src_dataset_prefix=src_file,
                  tgt_dataset_prefix=tgt_file,
                  src_tokenizer=self.encoder_tokenizer,
                  tgt_tokenizer=self.decoder_tokenizer,
                  max_src_seq_length=cfg.max_seq_length,
                  max_tgt_seq_length=cfg.max_seq_length,
                  max_num_samples=num_samples[0],
                  seed=self._cfg.seed,
                  prepend_id=prepend_id,
              )
          elif cfg.get("objective", "nmt") == "nmt-xlm":
              # Pass sentinel tokens to the dataset after removing language tokens.
              additional_special_ids = (
                  self.encoder_tokenizer.additional_special_tokens_ids
              )
              sentinel_tokens = [
                  id
                  for id in additional_special_ids
                  if id not in self.multilingual_ids
              ]
              dataset = (
                  BinarizedMemmapCrossLingualMLMAndTranslationDataset(
                      src_dataset_prefix=src_file,
                      tgt_dataset_prefix=tgt_file,
                      src_tokenizer=self.encoder_tokenizer,
                      tgt_tokenizer=self.decoder_tokenizer,
                      src_language=src_language,
                      tgt_language=tgt_language,
                      max_src_seq_length=cfg.max_seq_length // 2,
                      max_tgt_seq_length=cfg.max_seq_length // 2,
                      max_seq_length_dec=cfg.max_seq_length,
                      max_num_samples=num_samples[0],
                      sampling_ratios=cfg.sampling_ratios,
                      sentinel_tokens=sentinel_tokens,
                      seed=self._cfg.seed,
                  )
              )
      elif cfg.dataset_type == "text_memmap":
          if cfg.get("objective", "nmt") == "nmt":
              dataset = TextMemmapSequenceToSequenceDataset(
                  src_file_name=src_file,
                  tgt_file_name=tgt_file,
                  src_tokenizer=self.encoder_tokenizer,
                  tgt_tokenizer=self.decoder_tokenizer,
                  max_src_seq_length=cfg.max_seq_length,
                  max_tgt_seq_length=cfg.max_seq_length,
                  max_num_samples=num_samples[0],
                  seed=self._cfg.seed,
                  prepend_id=prepend_id,
              )
          elif cfg.get("objective", "nmt") == "nmt-xlm":
              additional_special_ids = (
                  self.encoder_tokenizer.additional_special_tokens_ids
              )
              sentinel_tokens = [
                  id
                  for id in additional_special_ids
                  if id not in self.multilingual_ids
              ]
              dataset = TextMemmapCrossLingualMLMAndTranslationDataset(
                  src_file_name=src_file,
                  tgt_file_name=tgt_file,
                  src_tokenizer=self.encoder_tokenizer,
                  tgt_tokenizer=self.decoder_tokenizer,
                  src_language=src_language,
                  tgt_language=tgt_language,
                  max_src_seq_length=cfg.max_seq_length // 2,
                  max_tgt_seq_length=cfg.max_seq_length // 2,
                  max_seq_length_dec=cfg.max_seq_length,
                  max_num_samples=num_samples[0],
                  sampling_ratios=cfg.sampling_ratios,
                  sentinel_tokens=sentinel_tokens,
                  seed=self._cfg.seed,
              )
  
      return dataset
  
imports: import enum
  import itertools
  import random
  from typing import List, Optional
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from omegaconf.listconfig import ListConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from sacrebleu import corpus_bleu
  from nemo.collections.nlp.data.common.sequence_to_sequence_dataset import BinarizedMemmapSequenceToSequenceDataset, TextMemmapSequenceToSequenceDataset
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers import MegatronPretrainingBatchSampler
  from nemo.collections.nlp.data.language_modeling.megatron.xlm_dataset import BinarizedMemmapCrossLingualMLMAndTranslationDataset, TextMemmapCrossLingualMLMAndTranslationDataset
  from nemo.collections.nlp.models.language_modeling.megatron_lm_encoder_decoder_model import MegatronLMEncoderDecoderModel
  from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model
  from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel
  from nemo.collections.nlp.modules.common.megatron.megatron_export import DecEmb, EncEmb, TokensHeadEmb
  from nemo.collections.nlp.modules.common.megatron.utils import get_iterator_k_split
  from nemo.collections.nlp.parts.nlp_overrides import GlobalBatchDataFetcher
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes import Exportable
  from nemo.utils import AppState, logging, timers
label_desc: Use image node_id 23 for calling the MegatronNMTModel obj's underlying member method code with example usage: obj._instantiate_memmap_dataset(cfg, src_file, tgt_file, src_language, tgt_language, num_samples, prepend_id) and returns: dataset