class_docstr: Task for training masked language models (e.g., BERT, RoBERTa).
class_docstr_tok: ["Task","for","training","masked","language","models","(","e.g.",",","BERT",",","RoBERTa",")","."]
docstr: Load a given dataset split.
  
  Args:
      split (str): name of the split (e.g., train, valid, test)
docstr_tok: ["Load","a","given","dataset","split",".","Args",":","split","(","str",")",":","name","of","the","split","(","e.g.",",","train",",","valid",",","test",")"]
code: def load_dataset(self, split, epoch=1, combine=False, **kwargs):
      """Load a given dataset split.
  
      Args:
          split (str): name of the split (e.g., train, valid, test)
      """
      paths = utils.split_paths(self.args.data)
      assert len(paths) > 0
      data_path = paths[(epoch - 1) % len(paths)]
  
      languages = sorted(
          name
          for name in os.listdir(data_path)
          if os.path.isdir(os.path.join(data_path, name))
      )
  
      logger.info(
          "Training on {0} languages: {1}".format(
              len(languages), languages
          )
      )
      logger.info(
          "Language to id mapping: ",
          {lang: id for id, lang in enumerate(languages)},
      )
  
      mask_whole_words = self._get_whole_word_mask()
      lang_datasets = []
      for lang_id, language in enumerate(languages):
          split_path = os.path.join(data_path, language, split)
  
          dataset = data_utils.load_indexed_dataset(
              split_path,
              self.source_dictionary,
              self.args.dataset_impl,
              combine=combine,
          )
          if dataset is None:
              raise FileNotFoundError(
                  "Dataset not found: {} ({})".format(split, split_path)
              )
  
          # create continuous blocks of tokens
          dataset = TokenBlockDataset(
              dataset,
              dataset.sizes,
              self.args.tokens_per_sample - 1,  # one less for <s>
              pad=self.source_dictionary.pad(),
              eos=self.source_dictionary.eos(),
              break_mode=self.args.sample_break_mode,
          )
          logger.info(
              "loaded {} blocks from: {}".format(
                  len(dataset), split_path
              )
          )
  
          # prepend beginning-of-sentence token (<s>, equiv. to [CLS] in BERT)
          dataset = PrependTokenDataset(
              dataset, self.source_dictionary.bos()
          )
  
          src_dataset, tgt_dataset = MaskTokensDataset.apply_mask(
              dataset,
              self.source_dictionary,
              pad_idx=self.source_dictionary.pad(),
              mask_idx=self.mask_idx,
              seed=self.args.seed,
              mask_prob=self.args.mask_prob,
              leave_unmasked_prob=self.args.leave_unmasked_prob,
              random_token_prob=self.args.random_token_prob,
              freq_weighted_replacement=self.args.freq_weighted_replacement,
              mask_whole_words=mask_whole_words,
          )
  
          lang_dataset = NestedDictionaryDataset(
              {
                  "net_input": {
                      "src_tokens": PadDataset(
                          src_dataset,
                          pad_idx=self.source_dictionary.pad(),
                          left_pad=False,
                      ),
                      "src_lengths": NumelDataset(
                          src_dataset, reduce=False
                      ),
                  },
                  "target": PadDataset(
                      tgt_dataset,
                      pad_idx=self.source_dictionary.pad(),
                      left_pad=False,
                  ),
                  "nsentences": NumSamplesDataset(),
                  "ntokens": NumelDataset(src_dataset, reduce=True),
                  "lang_id": RawLabelDataset(
                      [lang_id] * src_dataset.sizes.shape[0]
                  ),
              },
              sizes=[src_dataset.sizes],
          )
          lang_datasets.append(lang_dataset)
  
      dataset_lengths = np.array(
          [len(d) for d in lang_datasets],
          dtype=float,
      )
      logger.info(
          "loaded total {} blocks for all languages".format(
              dataset_lengths.sum(),
          )
      )
      if split == self.args.train_subset:
          # For train subset, additionally up or down sample languages.
          sample_probs = self._get_sample_prob(dataset_lengths)
          logger.info(
              "Sample probability by language: ",
              {
                  lang: "{0:.4f}".format(sample_probs[id])
                  for id, lang in enumerate(languages)
              },
          )
          size_ratio = (
              sample_probs * dataset_lengths.sum()
          ) / dataset_lengths
          logger.info(
              "Up/Down Sampling ratio by language: ",
              {
                  lang: "{0:.2f}".format(size_ratio[id])
                  for id, lang in enumerate(languages)
              },
          )
  
          resampled_lang_datasets = [
              ResamplingDataset(
                  lang_datasets[i],
                  size_ratio=size_ratio[i],
                  seed=self.args.seed,
                  epoch=epoch,
                  replace=size_ratio[i] >= 1.0,
              )
              for i, d in enumerate(lang_datasets)
          ]
          dataset = ConcatDataset(resampled_lang_datasets)
      else:
          dataset = ConcatDataset(lang_datasets)
          lang_splits = [split]
          for lang_id, lang_dataset in enumerate(lang_datasets):
              split_name = split + "_" + languages[lang_id]
              lang_splits.append(split_name)
              self.datasets[split_name] = lang_dataset
  
          # [TODO]: This is hacky for now to print validation ppl for each
          # language individually. Maybe need task API changes to allow it
          # in more generic ways.
          if split in self.args.valid_subset:
              self.args.valid_subset = self.args.valid_subset.replace(
                  split, ",".join(lang_splits)
              )
  
      with data_utils.numpy_seed(self.args.seed + epoch):
          shuffle = np.random.permutation(len(dataset))
  
      self.datasets[split] = SortDataset(
          dataset,
          sort_order=[
              shuffle,
              dataset.sizes,
          ],
      )
  
file_path: fairseq/fairseq/tasks/multilingual_masked_lm.py
imports: import logging
  import os
  import numpy
  import torch
  from fairseq import utils
  from fairseq.data import ConcatDataset, Dictionary, IdDataset, MaskTokensDataset, NestedDictionaryDataset, NumelDataset, NumSamplesDataset, PadDataset, PrependTokenDataset, RawLabelDataset, ResamplingDataset, SortDataset, TokenBlockDataset, data_utils, encoders
  from fairseq.tasks import LegacyFairseqTask, register_task
label_desc: Use image node_id 6 for calling the MultiLingualMaskedLMTask obj's underlying member method code with example usage: obj.load_dataset(split, epoch, combine) without return types