docstr: Load a dataset split.
docstr_tok: ["Load","a","dataset","split","."]
code: def load_dataset(
      self,
      split,
      src_bin_path,
      tgt_bin_path,
      forward_model=None,
      backward_model=None,
  ):
      """Load a dataset split."""
  
      corpus = ptt_data.ParallelCorpusConfig(
          source=ptt_data.CorpusConfig(
              dialect=self.source_lang, data_file=src_bin_path
          ),
          target=ptt_data.CorpusConfig(
              dialect=self.target_lang, data_file=tgt_bin_path
          ),
          weights_file=None,
      )
  
      if self.args.log_verbose:
          print("Starting to load binarized data files.", flush=True)
      ptt_data_utils.validate_corpus_exists(corpus=corpus, split=split)
  
      forward_tgt_dataset = (
          ptt_data.InMemoryIndexedDataset.create_from_file(
              corpus.target.data_file
          )
      )
      backward_tgt_dataset = (
          ptt_data.InMemoryIndexedDataset.create_from_file(
              corpus.source.data_file
          )
      )
      forward_src_dataset = (
          ptt_data.InMemoryIndexedDataset.create_from_file(
              corpus.source.data_file
          )
      )
      backward_src_dataset = (
          ptt_data.InMemoryIndexedDataset.create_from_file(
              corpus.target.data_file
          )
      )
      forward_parallel_dataset = (
          weighted_data.WeightedLanguagePairDataset(
              src=forward_src_dataset,
              src_sizes=forward_src_dataset.sizes,
              src_dict=self.source_dictionary,
              tgt=forward_tgt_dataset,
              tgt_sizes=forward_tgt_dataset.sizes,
              tgt_dict=self.target_dictionary,
              remove_eos_from_source=self.remove_eos_from_source,
              append_eos_to_target=True,
          )
      )
      backward_parallel_dataset = (
          weighted_data.WeightedLanguagePairDataset(
              src=backward_src_dataset,
              src_sizes=backward_src_dataset.sizes,
              src_dict=self.target_dictionary,
              tgt=backward_tgt_dataset,
              tgt_sizes=backward_tgt_dataset.sizes,
              tgt_dict=self.source_dictionary,
              remove_eos_from_source=self.remove_eos_from_source,
              append_eos_to_target=True,
          )
      )
  
      dataset_map = OrderedDict(
          [
              (
                  f"{self.source_lang}-{self.target_lang}",
                  forward_parallel_dataset,
              ),
              (
                  f"{self.target_lang}-{self.source_lang}",
                  backward_parallel_dataset,
              ),
          ]
      )
  
      assert (forward_model and backward_model) or (
          forward_model is None and backward_model is None
      ), (
          "Only one of forward or backward models can't be null;"
          " both have to be non-null or null"
      )
      if forward_model and backward_model:
          fwd_generator = beam_decode.SequenceGenerator(
              models=[forward_model], tgt_dict=self.source_dictionary
          )
          bwd_generator = beam_decode.SequenceGenerator(
              models=[backward_model], tgt_dict=self.target_dictionary
          )
  
          def monolingual_dataset(
              path,
              dictionary,
              is_source=False,
              num_examples_limit: Optional[int] = None,
          ):
              dataset = self.load_monolingual_dataset(
                  path,
                  is_source=is_source,
                  num_examples_limit=num_examples_limit,
              )
              return LanguagePairDataset(
                  src=dataset,
                  src_sizes=dataset.sizes,
                  src_dict=dictionary,
                  tgt=None,
                  tgt_sizes=None,
                  tgt_dict=None,
              )
  
          monolingual_num_examples_limit = None
          if self.args.monolingual_ratio is not None:
              monolingual_num_examples_limit = int(
                  self.args.monolingual_ratio
                  * len(forward_parallel_dataset)
              )
  
          src_dataset = monolingual_dataset(
              path=self.args.train_mono_source_binary_path,
              dictionary=self.source_dictionary,
              is_source=True,
              num_examples_limit=monolingual_num_examples_limit,
          )
          tgt_dataset = monolingual_dataset(
              path=self.args.train_mono_target_binary_path,
              dictionary=self.target_dictionary,
              is_source=False,
              num_examples_limit=monolingual_num_examples_limit,
          )
  
          def generate_fn(generator):
              def _generate_fn(sample):
                  net_input = sample["net_input"]
                  maxlen = int(
                      self.args.max_len_a
                      * net_input["src_tokens"].size(1)
                      + self.args.max_len_b
                  )
                  return generator.generate(net_input, maxlen=maxlen)
  
              return _generate_fn
  
          dataset_map[
              f"{self.source_lang}-"
              f"{self.target_lang}_{constants.MONOLINGUAL_DATA_IDENTIFIER}"
          ] = weighted_data.WeightedBacktranslationDataset(
              dataset=BacktranslationDataset(
                  tgt_dataset=TransformEosDataset(
                      dataset=tgt_dataset,
                      eos=self.target_dictionary.eos(),
                      # Remove EOS from the input before backtranslation.
                      remove_eos_from_src=True,
                  ),
                  src_dict=self.source_dictionary,
                  backtranslation_fn=generate_fn(bwd_generator),
                  output_collater=TransformEosDataset(
                      dataset=tgt_dataset,
                      eos=self.target_dictionary.eos(),
                      # The original input (now the target) doesn't have
                      # an EOS, so we need to add one. The generated
                      # backtranslation (now the source) will have an EOS,
                      # so we want to remove it.
                      append_eos_to_tgt=True,
                      remove_eos_from_src=True,
                  ).collater,
              )
          )
          dataset_map[
              f"{self.target_lang}-"
              f"{self.source_lang}_{constants.MONOLINGUAL_DATA_IDENTIFIER}"
          ] = weighted_data.WeightedBacktranslationDataset(
              dataset=BacktranslationDataset(
                  tgt_dataset=src_dataset,
                  src_dict=self.source_dictionary,
                  backtranslation_fn=generate_fn(fwd_generator),
                  output_collater=TransformEosDataset(
                      dataset=src_dataset,
                      eos=self.source_dictionary.eos(),
                      # The original input (now the target) doesn't have
                      # an EOS, so we need to add one. The generated
                      # backtranslation (now the source) will have an EOS,
                      # so we want to remove it.
                      append_eos_to_tgt=True,
                      remove_eos_from_src=True,
                  ).collater,
              )
          )
  
      # print before loading RoundRobinZipDatasets to help catch any bugs
      for dataset_key, dataset in dataset_map.items():
          print(
              f"| {split}: {dataset_key} {len(dataset)} examples in dataset"
          )
  
      self.datasets[split] = RoundRobinZipDatasets(dataset_map)
      print(
          f"| {split} {len(self.datasets[split])} examples in RoundRobinZipDatasets"
      )
  
      if self.args.log_verbose:
          print("Finished loading dataset", flush=True)
  
imports: import json
  from collections import OrderedDict
  from typing import Optional
  from fairseq import models
  from fairseq.data import BacktranslationDataset, LanguagePairDataset, RoundRobinZipDatasets, TransformEosDataset
  from fairseq.models import FairseqMultiModel
  from fairseq.tasks import register_task
  from pytorch_translate import beam_decode, constants, rnn
  from pytorch_translate.data import data, dictionary, utils, weighted_data
  from pytorch_translate.tasks.pytorch_translate_multi_task import PyTorchTranslateMultiTask
  from pytorch_translate.tasks.pytorch_translate_task import PytorchTranslateTask
label_desc: Use image node_id 6 for calling the PytorchTranslateSemiSupervised obj's underlying member method code with example usage: obj.load_dataset(split, src_bin_path, tgt_bin_path, forward_model, backward_model) and returns: LanguagePairDataset, _generate_fn, generator