class_docstr: Tasks store dictionaries and provide helpers for loading/iterating over
  Datasets, initializing the Model/Criterion and calculating the loss.
  
  Tasks have limited statefulness. In particular, state that needs to be
  saved to/loaded from checkpoints needs to be stored in the `self.state`
  :class:`StatefulContainer` object. For example::
  
      self.state.add_factory("dictionary", self.load_dictionary)
      print(self.state.dictionary)  # calls self.load_dictionary()
  
  This is necessary so that when loading checkpoints, we can properly
  recreate the task state after initializing the task instance.
class_docstr_tok: ["Tasks","store","dictionaries","and","provide","helpers","for","loading\/iterating","over","Datasets",",","initializing","the","Model\/Criterion","and","calculating","the","loss",".","Tasks","have","limited","statefulness",".","In","particular",",","state","that","needs","to","be","saved","to\/loaded","from","checkpoints","needs","to","be","stored","in","the","`","self.state","`",":","class",":","`","StatefulContainer","`","object",".","For","example",":",":","self.state.add_factory","(","``","dictionary","''",",","self.load_dictionary",")","print","(","self.state.dictionary",")","#","calls","self.load_dictionary","(",")","This","is","necessary","so","that","when","loading","checkpoints",",","we","can","properly","recreate","the","task","state","after","initializing","the","task","instance","."]
docstr: Get an iterator that yields batches of data from the given dataset.
  
  Args:
      dataset (~fairseq.data.FairseqDataset): dataset to batch
      max_tokens (int, optional): max number of tokens in each batch
          (default: None).
      max_sentences (int, optional): max number of sentences in each
          batch (default: None).
      max_positions (optional): max sentence length supported by the
          model (default: None).
      ignore_invalid_inputs (bool, optional): don't raise Exception for
          sentences that are too long (default: False).
      required_batch_size_multiple (int, optional): require batch size to
          be a multiple of N (default: 1).
      seed (int, optional): seed for random number generator for
          reproducibility (default: 1).
      num_shards (int, optional): shard the data iterator into N
          shards (default: 1).
      shard_id (int, optional): which shard of the data iterator to
          return (default: 0).
      num_workers (int, optional): how many subprocesses to use for data
          loading. 0 means the data will be loaded in the main process
          (default: 0).
      epoch (int, optional): the epoch to start the iterator from
          (default: 1).
      data_buffer_size (int, optional): number of batches to
          preload (default: 0).
      disable_iterator_cache (bool, optional): don't cache the
          EpochBatchIterator (ignores `FairseqTask::can_reuse_epoch_itr`)
          (default: False).
      skip_remainder_batch (bool, optional): if set, discard the last
          batch in each training epoch, as the last batch is often smaller than
              local_batch_size * distributed_word_size (default: ``True``).
      grouped_shuffling (bool, optional): group batches with each groups
          containing num_shards batches and shuffle groups. Reduces difference
          between sequence lengths among workers for batches sorted by length.
      update_epoch_batch_itr (bool optional): if true then donot use the cached
          batch iterator for the epoch
  
  Returns:
      ~fairseq.iterators.EpochBatchIterator: a batched iterator over the
          given dataset split
code: def get_batch_iterator(
      self,
      dataset,
      max_tokens=None,
      max_sentences=None,
      max_positions=None,
      ignore_invalid_inputs=False,
      required_batch_size_multiple=1,
      seed=1,
      num_shards=1,
      shard_id=0,
      num_workers=0,
      epoch=1,
      data_buffer_size=0,
      disable_iterator_cache=False,
      skip_remainder_batch=False,
      grouped_shuffling=False,
      update_epoch_batch_itr=False,
  ):
      """
      Get an iterator that yields batches of data from the given dataset.
  
      Args:
          dataset (~fairseq.data.FairseqDataset): dataset to batch
          max_tokens (int, optional): max number of tokens in each batch
              (default: None).
          max_sentences (int, optional): max number of sentences in each
              batch (default: None).
          max_positions (optional): max sentence length supported by the
              model (default: None).
          ignore_invalid_inputs (bool, optional): don't raise Exception for
              sentences that are too long (default: False).
          required_batch_size_multiple (int, optional): require batch size to
              be a multiple of N (default: 1).
          seed (int, optional): seed for random number generator for
              reproducibility (default: 1).
          num_shards (int, optional): shard the data iterator into N
              shards (default: 1).
          shard_id (int, optional): which shard of the data iterator to
              return (default: 0).
          num_workers (int, optional): how many subprocesses to use for data
              loading. 0 means the data will be loaded in the main process
              (default: 0).
          epoch (int, optional): the epoch to start the iterator from
              (default: 1).
          data_buffer_size (int, optional): number of batches to
              preload (default: 0).
          disable_iterator_cache (bool, optional): don't cache the
              EpochBatchIterator (ignores `FairseqTask::can_reuse_epoch_itr`)
              (default: False).
          skip_remainder_batch (bool, optional): if set, discard the last
              batch in each training epoch, as the last batch is often smaller than
                  local_batch_size * distributed_word_size (default: ``True``).
          grouped_shuffling (bool, optional): group batches with each groups
              containing num_shards batches and shuffle groups. Reduces difference
              between sequence lengths among workers for batches sorted by length.
          update_epoch_batch_itr (bool optional): if true then donot use the cached
              batch iterator for the epoch
  
      Returns:
          ~fairseq.iterators.EpochBatchIterator: a batched iterator over the
              given dataset split
      """
      can_reuse_epoch_itr = (
          not disable_iterator_cache
          and not update_epoch_batch_itr
          and self.can_reuse_epoch_itr(dataset)
      )
      logger.info(f"can_reuse_epoch_itr = {can_reuse_epoch_itr}")
      if can_reuse_epoch_itr and dataset in self.dataset_to_epoch_iter:
          logger.debug(
              "reusing EpochBatchIterator for epoch {}".format(epoch)
          )
          return self.dataset_to_epoch_iter[dataset]
  
      assert isinstance(dataset, FairseqDataset)
  
      # initialize the dataset with the correct starting epoch
      dataset.set_epoch(epoch)
  
      def make_batches(dataset, epoch):
          logger.info(f"creating new batches for epoch {epoch}")
  
          # get indices ordered by example size
          with data_utils.numpy_seed(seed + epoch):
              indices = dataset.ordered_indices()
  
          # filter examples that are too large
          if max_positions is not None:
              indices = self.filter_indices_by_size(
                  indices, dataset, max_positions, ignore_invalid_inputs
              )
  
          # create mini-batches with given size constraints
          batches = dataset.batch_by_size(
              indices,
              max_tokens=max_tokens,
              max_sentences=max_sentences,
              required_batch_size_multiple=required_batch_size_multiple,
          )
          return batches
  
      reuse_dataloader = getattr(self.cfg, "reuse_dataloader", True)
      persistent_workers = getattr(self.cfg, "persistent_workers", True)
      rebuild_batches = getattr(self.cfg, "rebuild_batches", False)
      logger.info(f"reuse_dataloader = {reuse_dataloader}")
      logger.info(f"rebuild_batches = {rebuild_batches}")
  
      if rebuild_batches:
          logger.info("batches will be rebuilt for each epoch")
          batch_sampler = make_batches
      else:
          batch_sampler = make_batches(dataset, epoch)
  
      # return a reusable, sharded iterator
      epoch_iter = iterators.EpochBatchIterator(
          dataset=dataset,
          collate_fn=dataset.collater,
          batch_sampler=batch_sampler,
          seed=seed,
          num_shards=num_shards,
          shard_id=shard_id,
          num_workers=num_workers,
          epoch=epoch,
          buffer_size=data_buffer_size,
          skip_remainder_batch=skip_remainder_batch,
          grouped_shuffling=grouped_shuffling,
          reuse_dataloader=reuse_dataloader,
          persistent_workers=persistent_workers,
      )
  
      if can_reuse_epoch_itr:
          self.dataset_to_epoch_iter[dataset] = epoch_iter
  
      return epoch_iter
  
imports: import logging
  import os
  import warnings
  from argparse import Namespace
  from typing import Any, Callable, Dict, List
  import torch
  from fairseq import search, tokenizer, utils
  from fairseq.logging import metrics
  from fairseq.data import Dictionary, FairseqDataset, data_utils, encoders, iterators
  from fairseq.dataclass import FairseqDataclass
  from fairseq.dataclass.utils import gen_parser_from_dataclass
  from fairseq.optim.amp_optimizer import AMPOptimizer
  from omegaconf import DictConfig
label_desc: Use image node_id 12 for calling the FairseqTask obj's underlying member method code with example usage: obj.get_batch_iterator(dataset, max_tokens, max_sentences, max_positions, ignore_invalid_inputs, required_batch_size_multiple, seed, num_shards, shard_id, num_workers, epoch, data_buffer_size, disable_iterator_cache, skip_remainder_batch, grouped_shuffling, update_epoch_batch_itr) and returns: epoch_iter, self, batches