class_docstr: Datapipe designed to operate over binary (npy) versions of Criteo datasets. Loads
  the entire dataset into memory to prevent disk speed from affecting throughout. Each
  rank reads only the data for the portion of the dataset it is responsible for.
  
  The torchrec/datasets/scripts/npy_preproc_criteo.py script can be used to convert
  the Criteo tsv files to the npy files expected by this dataset.
  
  Args:
      stage (str): "train", "val", or "test".
      dense_paths (List[str]): List of path strings to dense npy files.
      sparse_paths (List[str]): List of path strings to sparse npy files.
      labels_paths (List[str]): List of path strings to labels npy files.
      batch_size (int): batch size.
      rank (int): rank.
      world_size (int): world size.
      shuffle_batches (bool): Whether to shuffle batches
      hashes (Optional[int]): List of max categorical feature value for each feature.
          Length of this list should be CAT_FEATURE_COUNT.
      path_manager_key (str): Path manager key used to load from different
          filesystems.
  
  Example::
  
      template = "/home/datasets/criteo/1tb_binary/day_{}_{}.npy"
      datapipe = InMemoryBinaryCriteoIterDataPipe(
          dense_paths=[template.format(0, "dense"), template.format(1, "dense")],
          sparse_paths=[template.format(0, "sparse"), template.format(1, "sparse")],
          labels_paths=[template.format(0, "labels"), template.format(1, "labels")],
          batch_size=1024,
          rank=torch.distributed.get_rank(),
          world_size=torch.distributed.get_world_size(),
      )
      batch = next(iter(datapipe))
class_docstr_tok: ["Datapipe","designed","to","operate","over","binary","(","npy",")","versions","of","Criteo","datasets",".","Loads","the","entire","dataset","into","memory","to","prevent","disk","speed","from","affecting","throughout",".","Each","rank","reads","only","the","data","for","the","portion","of","the","dataset","it","is","responsible","for",".","The","torchrec\/datasets\/scripts\/npy_preproc_criteo.py","script","can","be","used","to","convert","the","Criteo","tsv","files","to","the","npy","files","expected","by","this","dataset",".","Args",":","stage","(","str",")",":","``","train","''",",","``","val","''",",","or","``","test","''",".","dense_paths","(","List","[","str","]",")",":","List","of","path","strings","to","dense","npy","files",".","sparse_paths","(","List","[","str","]",")",":","List","of","path","strings","to","sparse","npy","files",".","labels_paths","(","List","[","str","]",")",":","List","of","path","strings","to","labels","npy","files",".","batch_size","(","int",")",":","batch","size",".","rank","(","int",")",":","rank",".","world_size","(","int",")",":","world","size",".","shuffle_batches","(","bool",")",":","Whether","to","shuffle","batches","hashes","(","Optional","[","int","]",")",":","List","of","max","categorical","feature","value","for","each","feature",".","Length","of","this","list","should","be","CAT_FEATURE_COUNT",".","path_manager_key","(","str",")",":","Path","manager","key","used","to","load","from","different","filesystems",".","Example",":",":","template","=","``","\/home\/datasets\/criteo\/1tb_binary\/day_","{","}","_","{","}",".npy","''","datapipe","=","InMemoryBinaryCriteoIterDataPipe","(","dense_paths=","[","template.format","(","0",",","``","dense","''",")",",","template.format","(","1",",","``","dense","''",")","]",",","sparse_paths=","[","template.format","(","0",",","``","sparse","''",")",",","template.format","(","1",",","``","sparse","''",")","]",",","labels_paths=","[","template.format","(","0",",","``","labels","''",")",",","template.format","(","1",",","``","labels","''",")","]",",","batch_size=1024",",","rank=torch.distributed.get_rank","(",")",",","world_size=torch.distributed.get_world_size","(",")",",",")","batch","=","next","(","iter","(","datapipe",")",")"]
code: def _np_arrays_to_batch(
      self, dense: np.ndarray, sparse: np.ndarray, labels: np.ndarray
  ) -> Batch:
      if self.shuffle_batches:
          # Shuffle all 3 in unison
          shuffler = np.random.permutation(len(dense))
          dense = dense[shuffler]
          sparse = sparse[shuffler]
          labels = labels[shuffler]
  
      batch_size = len(dense)
      num_ids_in_batch = CAT_FEATURE_COUNT * batch_size
      if batch_size == self.batch_size:
          length_per_key = self.length_per_key
          offset_per_key = self.offset_per_key
      else:
          # handle last batch in dataset when it's an incomplete batch.
          length_per_key = CAT_FEATURE_COUNT * [batch_size]
          offset_per_key = [
              batch_size * i for i in range(CAT_FEATURE_COUNT + 1)
          ]
  
      return Batch(
          dense_features=torch.from_numpy(dense),
          sparse_features=KeyedJaggedTensor(
              keys=self.keys,
              # transpose + reshape(-1) incurs an additional copy.
              values=torch.from_numpy(
                  sparse.transpose(1, 0).reshape(-1)
              ),
              lengths=self.lengths[:num_ids_in_batch],
              offsets=self.offsets[: num_ids_in_batch + 1],
              stride=batch_size,
              length_per_key=length_per_key,
              offset_per_key=offset_per_key,
              index_per_key=self.index_per_key,
          ),
          labels=torch.from_numpy(labels.reshape(-1)),
      )
  
imports: import os
  import shutil
  import time
  from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
  import numpy
  import torch
  import torch.utils.data.datapipes
  from iopath.common.file_io import PathManager, PathManagerFactory
  from pyre_extensions import none_throws
  from torch.utils.data import IterableDataset, IterDataPipe
  from torchrec.datasets.utils import Batch, LoadFiles, PATH_MANAGER_KEY, ReadLinesFromCSV, safe_cast
  from torchrec.sparse.jagged_tensor import KeyedJaggedTensor
label_desc: Use image node_id 4 for calling the InMemoryBinaryCriteoIterDataPipe obj's underlying member method code with example usage: obj._np_arrays_to_batch(dense, sparse, labels) and returns: Batch