class_docstr: Megatron NMT training
class_docstr_tok: ["Megatron","NMT","training"]
docstr: Builds a memmap dataset from a existing binary based o nthe provided config.
code: def build_memmap_dataset_from_config(self, cfg: DictConfig):
      """Builds a memmap dataset from a existing binary based o nthe provided config."""
      is_src_listconfig = isinstance(cfg.src_file_name, ListConfig)
      is_tgt_listconfig = isinstance(cfg.tgt_file_name, ListConfig)
      # If multilingual, make sure both source and target are list configs
      if self.multilingual:
          if not (is_src_listconfig and is_tgt_listconfig):
              raise ValueError(
                  f"Multilingual datasets must be configured with a ListConfig for both src_file_name and tgt_file_name"
              )
      if (
          is_src_listconfig
          and not is_tgt_listconfig
          or is_tgt_listconfig
          and not is_src_listconfig
      ):
          raise ValueError(
              f"Datasets must be configured with a ListConfig for both src_file_name and tgt_file_name or neither. Found only one of them as listconfig."
          )
  
      if is_src_listconfig and is_tgt_listconfig:
          if len(cfg.src_file_name) != len(cfg.tgt_file_name):
              raise ValueError(
                  f"Datasets must have the same number of files in src_file_name and tgt_file_name"
              )
  
          if (
              cfg.concat_sampling_probabilities is None
              or not isinstance(
                  cfg.concat_sampling_probabilities, ListConfig
              )
          ):
              raise ValueError(
                  f"concat_sampling_probabilities must be a ListConfig with the same number of files in src_file_name and tgt_file_name, found {cfg.concat_sampling_probabilities}"
              )
          if len(cfg.concat_sampling_probabilities) != len(
              cfg.src_file_name
          ):
              raise ValueError(
                  f"concat_sampling_probabilities must be of the same size as src_file_name and tgt_file_name. Provided size {len(cfg.concat_sampling_probabilities)}, number of datasets {len(cfg.src_file_name)}"
              )
  
          # Construct the data prefix list for `get_datasets_weights_and_num_samples()` that is of the format [weight1,file_name1,weight2,file_name2,...]
          data_prefix = []
          for weight, prefix in zip(
              cfg.concat_sampling_probabilities, cfg.src_file_name
          ):
              data_prefix.append(weight)
              data_prefix.append(prefix)
  
          num_train_samples = [
              self.trainer.max_steps * self._cfg.global_batch_size
          ]
          (
              _,
              _,
              num_train_samples_per_dataset,
          ) = get_datasets_weights_and_num_samples(
              data_prefix, num_train_samples
          )
          num_train_samples_after_blend = sum(
              [x[0] for x in num_train_samples_per_dataset]
          )
  
          datasets = []
  
          # For many -> one multilingual or bilingual models, we don't need to prepend a language token ID
          if (
              not self.multilingual
              or self.multilingual_type
              == MultilingualModelType.many_to_one
          ):
              multilingual_ids = [None] * len(cfg.src_file_name)
          # For one -> many and many -> many multilingual models, we need to prepend a language token ID
          else:
              multilingual_ids = [
                  self.multilingual_lang_to_id[lang]
                  for lang in self.cfg.tgt_language
              ]
  
          for idx, (src_file, tgt_file, num_samples) in enumerate(
              zip(
                  cfg.src_file_name,
                  cfg.tgt_file_name,
                  num_train_samples_per_dataset,
              )
          ):
              dataset = self._instantiate_memmap_dataset(
                  cfg=cfg,
                  src_file=src_file,
                  tgt_file=tgt_file,
                  num_samples=num_samples,
                  prepend_id=multilingual_ids[idx],
                  src_language=self.src_language
                  if not isinstance(self.src_language, ListConfig)
                  else self.src_language[idx],
                  tgt_language=self.tgt_language
                  if not isinstance(self.tgt_language, ListConfig)
                  else self.tgt_language[idx],
              )
              datasets.append(dataset)
          dataset = BlendableDataset(
              datasets=datasets,
              weights=cfg.concat_sampling_probabilities,
              size=num_train_samples_after_blend,
          )
      else:
          dataset = self._instantiate_memmap_dataset(
              cfg=cfg,
              src_file=cfg.src_file_name,
              tgt_file=cfg.tgt_file_name,
              num_samples=[
                  self.trainer.max_steps * self._cfg.global_batch_size
              ],
              src_language=self.src_language,
              tgt_language=self.tgt_language,
          )
      return dataset
  
imports: import enum
  import itertools
  import random
  from typing import List, Optional
  import numpy
  import torch
  from omegaconf.dictconfig import DictConfig
  from omegaconf.listconfig import ListConfig
  from pytorch_lightning.trainer.trainer import Trainer
  from sacrebleu import corpus_bleu
  from nemo.collections.nlp.data.common.sequence_to_sequence_dataset import BinarizedMemmapSequenceToSequenceDataset, TextMemmapSequenceToSequenceDataset
  from nemo.collections.nlp.data.language_modeling.megatron.base_dataset_utils import get_datasets_weights_and_num_samples
  from nemo.collections.nlp.data.language_modeling.megatron.blendable_dataset import BlendableDataset
  from nemo.collections.nlp.data.language_modeling.megatron.megatron_batch_samplers import MegatronPretrainingBatchSampler
  from nemo.collections.nlp.data.language_modeling.megatron.xlm_dataset import BinarizedMemmapCrossLingualMLMAndTranslationDataset, TextMemmapCrossLingualMLMAndTranslationDataset
  from nemo.collections.nlp.models.language_modeling.megatron_lm_encoder_decoder_model import MegatronLMEncoderDecoderModel
  from nemo.collections.nlp.models.language_modeling.megatron_t5_model import MegatronT5Model
  from nemo.collections.nlp.models.machine_translation.mt_enc_dec_model import MTEncDecModel
  from nemo.collections.nlp.modules.common.megatron.megatron_export import DecEmb, EncEmb, TokensHeadEmb
  from nemo.collections.nlp.modules.common.megatron.utils import get_iterator_k_split
  from nemo.collections.nlp.parts.nlp_overrides import GlobalBatchDataFetcher
  from nemo.collections.nlp.parts.utils_funcs import get_last_rank
  from nemo.core.classes import Exportable
  from nemo.utils import AppState, logging, timers
label_desc: Use image node_id 24 for calling the MegatronNMTModel obj's underlying member method code with example usage: obj.build_memmap_dataset_from_config(cfg) and returns: dataset