code: def load_langpair_dataset(
      data_path,
      split,
      src,
      src_dict,
      tgt,
      tgt_dict,
      combine,
      dataset_impl,
      upsample_primary,
      left_pad_source,
      left_pad_target,
      max_source_positions,
      max_target_positions,
      prepend_bos=False,
      load_alignments=False,
      truncate_source=False,
      append_source_id=False,
      num_buckets=0,
      shuffle=True,
      pad_to_multiple=1,
      shard_iterator=None,
      is_train=False,
  ):
      src_datasets = []
      tgt_datasets = []
      datasets_size = 0
      cur_pace, total_pace, itr = 1, 1, (0,)
      if is_train:
          cur_pace, total_pace = shard_iterator.pace()
          itr = shard_iterator.next()
      for k in itr:
          split_k = split + (str(k) if k > 0 else "")
          # infer langcode
          if split_exists(
              split_k, src, tgt, src, data_path, dataset_impl
          ):
              prefix = os.path.join(
                  data_path, "{}.{}-{}.".format(split_k, src, tgt)
              )
          elif split_exists(
              split_k, tgt, src, src, data_path, dataset_impl
          ):
              prefix = os.path.join(
                  data_path, "{}.{}-{}.".format(split_k, tgt, src)
              )
          else:
              raise FileNotFoundError(
                  "Dataset not found: {} ({})".format(
                      split_k, data_path
                  )
              )
  
          src_dataset = data_utils.load_indexed_dataset(
              prefix + src, src_dict, dataset_impl
          )
          if truncate_source:
              src_dataset = AppendTokenDataset(
                  TruncateDataset(
                      StripTokenDataset(src_dataset, src_dict.eos()),
                      max_source_positions - 1,
                  ),
                  src_dict.eos(),
              )
          src_datasets.append(src_dataset)
  
          tgt_dataset = data_utils.load_indexed_dataset(
              prefix + tgt, tgt_dict, dataset_impl
          )
          if tgt_dataset is not None:
              tgt_datasets.append(tgt_dataset)
          datasets_size += len(src_datasets[-1])
          if not combine:
              break
  
      logger.info(
          "{} {}/{} {}-{} {} examples".format(
              data_path, cur_pace, total_pace, src, tgt, datasets_size
          )
      )
      assert (
          len(src_datasets) == len(tgt_datasets)
          or len(tgt_datasets) == 0
      )
  
      if len(src_datasets) == 1:
          src_dataset = src_datasets[0]
          tgt_dataset = (
              tgt_datasets[0] if len(tgt_datasets) > 0 else None
          )
      else:
          sample_ratios = [1] * len(src_datasets)
          sample_ratios[0] = upsample_primary
          src_dataset = ConcatDataset(src_datasets, sample_ratios)
          if len(tgt_datasets) > 0:
              tgt_dataset = ConcatDataset(tgt_datasets, sample_ratios)
          else:
              tgt_dataset = None
  
      if prepend_bos:
          assert hasattr(src_dict, "bos_index") and hasattr(
              tgt_dict, "bos_index"
          )
          src_dataset = PrependTokenDataset(src_dataset, src_dict.bos())
          if tgt_dataset is not None:
              tgt_dataset = PrependTokenDataset(
                  tgt_dataset, tgt_dict.bos()
              )
  
      eos = None
      if append_source_id:
          src_dataset = AppendTokenDataset(
              src_dataset, src_dict.index("[{}]".format(src))
          )
          if tgt_dataset is not None:
              tgt_dataset = AppendTokenDataset(
                  tgt_dataset, tgt_dict.index("[{}]".format(tgt))
              )
          eos = tgt_dict.index("[{}]".format(tgt))
  
      align_dataset = None
      if load_alignments:
          align_path = os.path.join(
              data_path, "{}.align.{}-{}".format(split, src, tgt)
          )
          if indexed_dataset.dataset_exists(
              align_path, impl=dataset_impl
          ):
              align_dataset = data_utils.load_indexed_dataset(
                  align_path, None, dataset_impl
              )
  
      tgt_dataset_sizes = (
          tgt_dataset.sizes if tgt_dataset is not None else None
      )
      return LanguagePairDataset(
          src_dataset,
          src_dataset.sizes,
          src_dict,
          tgt_dataset,
          tgt_dataset_sizes,
          tgt_dict,
          left_pad_source=left_pad_source,
          left_pad_target=left_pad_target,
          align_dataset=align_dataset,
          eos=eos,
          num_buckets=num_buckets,
          shuffle=shuffle,
          pad_to_multiple=pad_to_multiple,
      )
  
imports: import itertools
  import json
  import logging
  import os
  import shutil
  import subprocess
  from pathlib import Path
  from argparse import Namespace
  import numpy
  from fairseq import metrics, options, utils
  from fairseq.data import AppendTokenDataset, ConcatDataset, LanguagePairDataset, PrependTokenDataset, StripTokenDataset, TruncateDataset, data_utils, encoders, indexed_dataset
  import torch.distributed
  from fairseq.tasks import LegacyFairseqTask, register_task
  from fairseq.tasks.translation import TranslationTask
label_desc: Use image node_id 7 for calling a global function with example usage: load_langpair_dataset(data_path, split, src, src_dict, tgt, tgt_dict, combine, dataset_impl, upsample_primary, left_pad_source, left_pad_target, max_source_positions, max_target_positions, prepend_bos, load_alignments, truncate_source, append_source_id, num_buckets, shuffle, pad_to_multiple, shard_iterator, is_train) and returns: LanguagePairDataset