class_docstr: Utility functions used to preprocess, save, load, partition, etc. the Criteo
  dataset in a binary (numpy) format.
class_docstr_tok: ["Utility","functions","used","to","preprocess",",","save",",","load",",","partition",",","etc",".","the","Criteo","dataset","in","a","binary","(","numpy",")","format","."]
docstr: Shuffle the dataset. Expects the files to be in .npy format and the data
  to be split by day and by dense, sparse and label data.
  Dense data must be in: day_x_dense.npy
  Sparse data must be in: day_x_sparse.npy
  Labels data must be in: day_x_labels.npy
  
  The dataset will be reconstructed, shuffled and then split back into
  separate dense, sparse and labels files.
  
  This will only shuffle the first DAYS-1 days as the training set. The final day will remain
  untouched as the validation, and training set.
  
  Args:
      input_dir_labels_and_dense (str): Input directory of labels and dense npy files.
      input_dir_sparse (str): Input directory of sparse npy files.
      output_dir_shuffled (str): Output directory for shuffled labels, dense and sparse npy files.
      rows_per_day Dict[int, int]: Number of rows in each file.
      output_dir_full_set (str): Output directory of the full dataset, if desired.
      days (int): Number of day files.
      int_columns (int): Number of columns with dense features.
      sparse_columns (int): Total number of categorical columns.
      path_manager_key (str): Path manager key used to load from different filesystems.
      random_seed (int): Random seed used for the random.shuffle operator.
docstr_tok: ["Shuffle","the","dataset",".","Expects","the","files","to","be","in",".npy","format","and","the","data","to","be","split","by","day","and","by","dense",",","sparse","and","label","data",".","Dense","data","must","be","in",":","day_x_dense.npy","Sparse","data","must","be","in",":","day_x_sparse.npy","Labels","data","must","be","in",":","day_x_labels.npy","The","dataset","will","be","reconstructed",",","shuffled","and","then","split","back","into","separate","dense",",","sparse","and","labels","files",".","This","will","only","shuffle","the","first","DAYS-1","days","as","the","training","set",".","The","final","day","will","remain","untouched","as","the","validation",",","and","training","set",".","Args",":","input_dir_labels_and_dense","(","str",")",":","Input","directory","of","labels","and","dense","npy","files",".","input_dir_sparse","(","str",")",":","Input","directory","of","sparse","npy","files",".","output_dir_shuffled","(","str",")",":","Output","directory","for","shuffled","labels",",","dense","and","sparse","npy","files",".","rows_per_day","Dict","[","int",",","int","]",":","Number","of","rows","in","each","file",".","output_dir_full_set","(","str",")",":","Output","directory","of","the","full","dataset",",","if","desired",".","days","(","int",")",":","Number","of","day","files",".","int_columns","(","int",")",":","Number","of","columns","with","dense","features",".","sparse_columns","(","int",")",":","Total","number","of","categorical","columns",".","path_manager_key","(","str",")",":","Path","manager","key","used","to","load","from","different","filesystems",".","random_seed","(","int",")",":","Random","seed","used","for","the","random.shuffle","operator","."]
code: def shuffle(
      input_dir_labels_and_dense: str,
      input_dir_sparse: str,
      output_dir_shuffled: str,
      rows_per_day: Dict[int, int],
      output_dir_full_set: Optional[str] = None,
      days: int = DAYS,
      int_columns: int = INT_FEATURE_COUNT,
      sparse_columns: int = CAT_FEATURE_COUNT,
      path_manager_key: str = PATH_MANAGER_KEY,
      random_seed: int = 0,
  ) -> None:
      """
      Shuffle the dataset. Expects the files to be in .npy format and the data
      to be split by day and by dense, sparse and label data.
      Dense data must be in: day_x_dense.npy
      Sparse data must be in: day_x_sparse.npy
      Labels data must be in: day_x_labels.npy
  
      The dataset will be reconstructed, shuffled and then split back into
      separate dense, sparse and labels files.
  
      This will only shuffle the first DAYS-1 days as the training set. The final day will remain
      untouched as the validation, and training set.
  
      Args:
          input_dir_labels_and_dense (str): Input directory of labels and dense npy files.
          input_dir_sparse (str): Input directory of sparse npy files.
          output_dir_shuffled (str): Output directory for shuffled labels, dense and sparse npy files.
          rows_per_day Dict[int, int]: Number of rows in each file.
          output_dir_full_set (str): Output directory of the full dataset, if desired.
          days (int): Number of day files.
          int_columns (int): Number of columns with dense features.
          sparse_columns (int): Total number of categorical columns.
          path_manager_key (str): Path manager key used to load from different filesystems.
          random_seed (int): Random seed used for the random.shuffle operator.
      """
  
      total_rows = sum(rows_per_day.values())
      columns = (
          int_columns + sparse_columns + 1
      )  # add 1 for label column
      full_dataset = np.zeros((total_rows, columns), dtype=np.float32)
      curr_first_row = 0
      curr_last_row = 0
      for d in range(0, days - 1):
          curr_last_row += rows_per_day[d]
  
          # dense
          path_to_file = os.path.join(
              input_dir_labels_and_dense, f"day_{d}_dense.npy"
          )
          data = np.load(path_to_file)
          print(
              f"Day {d} dense- {curr_first_row}-{curr_last_row} loaded files - {time.time()} - {path_to_file}"
          )
  
          full_dataset[
              curr_first_row:curr_last_row, 0:int_columns
          ] = data
          del data
  
          # sparse
          path_to_file = os.path.join(
              input_dir_sparse, f"day_{d}_sparse.npy"
          )
          data = np.load(path_to_file)
          print(
              f"Day {d} sparse- {curr_first_row}-{curr_last_row} loaded files - {time.time()} - {path_to_file}"
          )
  
          full_dataset[
              curr_first_row:curr_last_row, int_columns : columns - 1
          ] = data
          del data
  
          # labels
          path_to_file = os.path.join(
              input_dir_labels_and_dense, f"day_{d}_labels.npy"
          )
          data = np.load(path_to_file)
          print(
              f"Day {d} labels- {curr_first_row}-{curr_last_row} loaded files - {time.time()} - {path_to_file}"
          )
  
          full_dataset[
              curr_first_row:curr_last_row, columns - 1 :
          ] = data
          del data
  
          curr_first_row = curr_last_row
  
      path_manager = PathManagerFactory().get(path_manager_key)
  
      # Save the full dataset
      if output_dir_full_set is not None:
          full_output_file = os.path.join(
              output_dir_full_set, "full.npy"
          )
          with path_manager.open(full_output_file, "wb") as fout:
              print(f"Writing full set file: {full_output_file}")
              np.save(fout, full_dataset)
  
      print(f"Shuffling dataset with random_seed={random_seed}")
      np.random.seed(random_seed)
      np.random.shuffle(full_dataset)
  
      # Slice and save each portion into dense, sparse and labels
      curr_first_row = 0
      curr_last_row = 0
      for d in range(0, days - 1):
          curr_last_row += rows_per_day[d]
  
          # write dense columns
          shuffled_dense_file = os.path.join(
              output_dir_shuffled, f"day_{d}_dense.npy"
          )
          with path_manager.open(shuffled_dense_file, "wb") as fout:
              print(
                  f"Writing rows {curr_first_row}-{curr_last_row-1} dense file: {shuffled_dense_file}"
              )
              np.save(
                  fout,
                  full_dataset[
                      curr_first_row:curr_last_row, 0:int_columns
                  ],
              )
  
          # write sparse columns
          shuffled_sparse_file = os.path.join(
              output_dir_shuffled, f"day_{d}_sparse.npy"
          )
          with path_manager.open(shuffled_sparse_file, "wb") as fout:
              print(
                  f"Writing rows {curr_first_row}-{curr_last_row-1} sparse file: {shuffled_sparse_file}"
              )
              np.save(
                  fout,
                  full_dataset[
                      curr_first_row:curr_last_row,
                      int_columns : columns - 1,
                  ].astype(np.int32),
              )
  
          # write labels columns
          shuffled_labels_file = os.path.join(
              output_dir_shuffled, f"day_{d}_labels.npy"
          )
          with path_manager.open(shuffled_labels_file, "wb") as fout:
              print(
                  f"Writing rows {curr_first_row}-{curr_last_row-1} labels file: {shuffled_labels_file}"
              )
              np.save(
                  fout,
                  full_dataset[
                      curr_first_row:curr_last_row, columns - 1 :
                  ].astype(np.int32),
              )
          curr_first_row = curr_last_row
  
      # Directly copy over the last day's files since they will be used for validation and testing.
      for part, input_dir in [
          ("sparse", input_dir_sparse),
          ("dense", input_dir_labels_and_dense),
          ("labels", input_dir_labels_and_dense),
      ]:
          path_to_original = os.path.join(
              input_dir, f"day_{days-1}_{part}.npy"
          )
          val_train_path = os.path.join(
              output_dir_shuffled, f"day_{days-1}_{part}.npy"
          )
          shutil.copyfile(path_to_original, val_train_path)
          print(f"Copying over {path_to_original} to {val_train_path}")
  
imports: import os
  import shutil
  import time
  from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union
  import numpy
  import torch
  import torch.utils.data.datapipes
  from iopath.common.file_io import PathManager, PathManagerFactory
  from pyre_extensions import none_throws
  from torch.utils.data import IterableDataset, IterDataPipe
  from torchrec.datasets.utils import Batch, LoadFiles, PATH_MANAGER_KEY, ReadLinesFromCSV, safe_cast
  from torchrec.sparse.jagged_tensor import KeyedJaggedTensor
label_desc: Use image node_id 6 for calling the BinaryCriteoUtils obj's underlying member method code with example usage: obj.shuffle(input_dir_labels_and_dense, input_dir_sparse, output_dir_shuffled, rows_per_day, output_dir_full_set, days, int_columns, sparse_columns, path_manager_key, random_seed) without return types