code: def prepare_pretraining_dataloader(
      dataset, tokenizer, cfg_train, cfg_impl
  ):
      num_workers = get_num_workers(cfg_impl)
      if cfg_train.objective.name == "masked-lm":
          collate_fn = PatchedDataCollatorForLanguageModeling(
              tokenizer=tokenizer,
              mlm=not cfg_train.objective.disable_mlm,
              mlm_probability=cfg_train.objective.mlm_probability,
              pad_to_multiple_of=8,
              use_80_20_rule=cfg_train.objective.use_80_20_rule,
              token_drop=cfg_train.objective.token_drop,
          )
      else:
          collate_fn = None
  
      if isinstance(dataset, torch.utils.data.IterableDataset):
          # streaming mode for ready-made datasets, speed not tested
          if torch.distributed.is_initialized():
              dataset = split_dataset_by_node(
                  dataset,
                  rank=int(os.environ["RANK"]),
                  world_size=int(os.environ["WORLD_SIZE"]),
              )
  
          if cfg_impl.shuffle_in_dataloader:
              dataset = dataset.shuffle(seed=42, buffer_size=256)
          else:
              num_workers = 1  # ordered data is not loaded correctly with multiple workers in this case
          if cfg_train.reverse_dataset_order:
              raise ValueError("Reverse stream not implemented.")
          sampler = None
      else:
          # Normally, we'd just use nice map-style datasets:
          if torch.distributed.is_initialized():
              sampler = torch.utils.data.distributed.DistributedSampler(
                  dataset,
                  shuffle=cfg_impl.shuffle_in_dataloader,
                  drop_last=True,
              )
          else:
              if cfg_impl.shuffle_in_dataloader:
                  sampler = torch.utils.data.RandomSampler(dataset)
              else:
                  sampler = torch.utils.data.SequentialSampler(dataset)
          if cfg_train.reverse_dataset_order:
              dataset = dataset.select(reversed(range(len(dataset))))
  
      repeated_dataloader = InfiniteDataLoader(
          dataset,
          sampler=sampler,
          batch_size=cfg_impl.microbatch_size,
          num_workers=num_workers,
          pin_memory=cfg_impl.pin_memory,
          drop_last=True,
          prefetch_factor=cfg_impl.prefetch_factor
          if num_workers > 0
          else None,
          persistent_workers=cfg_impl.persistent_workers
          if num_workers > 0
          else False,
          collate_fn=collate_fn,
      )
      return repeated_dataloader
  
imports: import os
  import torch
  import transformers
  from torch.utils.data import DataLoader
  from datasets.distributed import split_dataset_by_node
label_desc: Use image node_id 5 for calling a global function with example usage: prepare_pretraining_dataloader(dataset, tokenizer, cfg_train, cfg_impl) and returns: repeated_dataloader