class_docstr: Batch Sampler class for multipack
class_docstr_tok: ["Batch","Sampler","class","for","multipack"]
code: def generate_batches(self, set_stats=False):
      indices = [idx for idx in self.sampler]
  
      lengths = self.lengths[indices]
      lengths_cumsum = np.cumsum(lengths)
  
      batches, total_used, total_slots = allocate(
          lengths=lengths,
          lengths_cumsum=lengths_cumsum,
          rank=0,
          c=self.batch_max_len,
          n=1,
      )
  
      batches = [
          [indices[b_idx] for b_idx in batch] for batch in batches
      ]
  
      # statistics
      if set_stats:
          self.eff_total_used += total_used
          self.eff_total_slots += total_slots
  
      return batches
  
file_path: axolotl/src/axolotl/utils/samplers/multipack.py
imports: import logging
  import math
  import os
  from typing import Any, Iterable, List, Union
  import numba
  import numpy
  from torch.utils.data import BatchSampler, Sampler
label_desc: Use image node_id 3 for calling the MultipackBatchSampler obj's underlying member method code with example usage: obj.generate_batches(set_stats) and returns: batches