class_docstr: Translation rerank task.
  The input can be either (src, tgt) sentence pairs or tgt sentence only.
class_docstr_tok: ["Translation","rerank","task",".","The","input","can","be","either","(","src",",","tgt",")","sentence","pairs","or","tgt","sentence","only","."]
docstr: Load a given dataset split (e.g., train, valid, test).
docstr_tok: ["Load","a","given","dataset","split","(","e.g.",",","train",",","valid",",","test",")","."]
code: def load_dataset(self, split, epoch=0, combine=False, **kwargs):
      """Load a given dataset split (e.g., train, valid, test)."""
      if self.cfg.data.endswith("1"):
          data_shard = (epoch - 1) % self.cfg.num_data_splits + 1
          data_path = self.cfg.data[:-1] + str(data_shard)
      else:
          data_path = self.cfg.data
  
      def get_path(type, data_split):
          return os.path.join(data_path, str(type), data_split)
  
      def make_dataset(type, dictionary, data_split, combine):
          split_path = get_path(type, data_split)
  
          dataset = data_utils.load_indexed_dataset(
              split_path,
              dictionary,
              combine=combine,
          )
          return dataset
  
      def load_split(data_split, metric):
          input_src = None
          if self.cfg.include_src:
              input_src = make_dataset(
                  "input_src",
                  self.dictionary,
                  data_split,
                  combine=False,
              )
              assert (
                  input_src is not None
              ), "could not find dataset: {}".format(
                  get_path("input_src", data_split)
              )
  
          input_tgt = make_dataset(
              "input_tgt", self.dictionary, data_split, combine=False
          )
          assert (
              input_tgt is not None
          ), "could not find dataset: {}".format(
              get_path("input_tgt", data_split)
          )
  
          label_path = f"{get_path(metric, data_split)}.{metric}"
          assert os.path.exists(
              label_path
          ), f"could not find dataset: {label_path}"
  
          np_labels = np.loadtxt(label_path)
          if self.cfg.target_metric == "ter":
              np_labels = -np_labels
          label = RawLabelDataset(np_labels)
  
          return input_src, input_tgt, label
  
      src_datasets = []
      tgt_datasets = []
      label_datasets = []
  
      if split == self.cfg.train_subset:
          for k in itertools.count():
              split_k = "train" + (str(k) if k > 0 else "")
              prefix = os.path.join(data_path, "input_tgt", split_k)
              if not indexed_dataset.dataset_exists(prefix, impl=None):
                  if k > 0:
                      break
                  else:
                      raise FileNotFoundError(
                          f"Dataset not found: {prefix}"
                      )
              input_src, input_tgt, label = load_split(
                  split_k, self.cfg.target_metric
              )
              src_datasets.append(input_src)
              tgt_datasets.append(input_tgt)
              label_datasets.append(label)
      else:
          input_src, input_tgt, label = load_split(
              split, self.cfg.target_metric
          )
          src_datasets.append(input_src)
          tgt_datasets.append(input_tgt)
          label_datasets.append(label)
  
      if len(tgt_datasets) == 1:
          input_tgt, label = tgt_datasets[0], label_datasets[0]
          if self.cfg.include_src:
              input_src = src_datasets[0]
      else:
          input_tgt = ConcatDataset(tgt_datasets)
          label = ConcatDataset(label_datasets)
          if self.cfg.include_src:
              input_src = ConcatDataset(src_datasets)
  
      input_tgt = TruncateDataset(input_tgt, self.cfg.max_positions)
      if self.cfg.include_src:
          input_src = PrependTokenDataset(
              input_src, self.dictionary.bos()
          )
          input_src = TruncateDataset(input_src, self.cfg.max_positions)
          src_lengths = NumelDataset(input_src, reduce=False)
          src_tokens = ConcatSentencesDataset(input_src, input_tgt)
      else:
          src_tokens = PrependTokenDataset(
              input_tgt, self.dictionary.bos()
          )
          src_lengths = NumelDataset(src_tokens, reduce=False)
  
      dataset = {
          "id": IdDataset(),
          "net_input": {
              "src_tokens": RightPadDataset(
                  src_tokens,
                  pad_idx=self.source_dictionary.pad(),
              ),
              "src_lengths": src_lengths,
          },
          "nsentences": NumSamplesDataset(),
          "ntokens": NumelDataset(src_tokens, reduce=True),
          "target": label,
      }
  
      dataset = NestedDictionaryDataset(
          dataset,
          sizes=[src_tokens.sizes],
      )
  
      assert (
          len(dataset) % self.cfg.mt_beam == 0
      ), "dataset size (%d) is not a multiple of beam size (%d)" % (
          len(dataset),
          self.cfg.mt_beam,
      )
  
      # no need to shuffle valid/test sets
      if not self.cfg.no_shuffle and split == self.cfg.train_subset:
          # need to keep all hypothese together
          start_idx = np.arange(0, len(dataset), self.cfg.mt_beam)
          with data_utils.numpy_seed(self.cfg.seed + epoch):
              np.random.shuffle(start_idx)
  
          idx = np.arange(0, self.cfg.mt_beam)
          shuffle = np.tile(idx, (len(start_idx), 1)).reshape(
              -1
          ) + np.tile(
              start_idx, (self.cfg.mt_beam, 1)
          ).transpose().reshape(
              -1
          )
  
          dataset = SortDataset(
              dataset,
              sort_order=[shuffle],
          )
  
      logger.info(f"Loaded {split} with #samples: {len(dataset)}")
  
      self.datasets[split] = dataset
      return self.datasets[split]
  
imports: from dataclasses import dataclass, field
  import itertools
  import logging
  import os
  import numpy
  import torch
  from fairseq.logging import metrics
  from fairseq.data import ConcatDataset, ConcatSentencesDataset, data_utils, Dictionary, IdDataset, indexed_dataset, NestedDictionaryDataset, NumSamplesDataset, NumelDataset, PrependTokenDataset, RawLabelDataset, RightPadDataset, SortDataset, TruncateDataset, TokenBlockDataset
  from fairseq.dataclass import ChoiceEnum, FairseqDataclass
  from fairseq.tasks import FairseqTask, register_task
  from omegaconf import II, MISSING
label_desc: Use image node_id 4 for calling the DiscriminativeRerankingNMTTask obj's underlying member method code with example usage: obj.load_dataset(split, epoch, combine) and returns: self, os, dataset, input_src,  input_tgt,  label