class_docstr: The streaming dataset can be used once your data have been optimised using the DatasetOptimiser class.
class_docstr_tok: ["The","streaming","dataset","can","be","used","once","your","data","have","been","optimised","using","the","DatasetOptimiser","class","."]
docstr: The streaming dataset can be used once your data have been optimised using the DatasetOptimiser class.
  
  Arguments:
      input_dir: Path to the folder where the input data is stored.
      item_loader: The logic to load an item from a chunk.
      shuffle: Whether to shuffle the data.
      drop_last: If `True`, drops the last items to ensure that
          all processes/workers return the same amount of data.
      seed: Random seed for shuffling.
      serializers: The serializers used to serialize and deserialize the chunks.
      max_cache_size: The maximum cache size used by the StreamingDataset.
docstr_tok: ["The","streaming","dataset","can","be","used","once","your","data","have","been","optimised","using","the","DatasetOptimiser","class",".","Arguments",":","input_dir",":","Path","to","the","folder","where","the","input","data","is","stored",".","item_loader",":","The","logic","to","load","an","item","from","a","chunk",".","shuffle",":","Whether","to","shuffle","the","data",".","drop_last",":","If","`","True","`",",","drops","the","last","items","to","ensure","that","all","processes\/workers","return","the","same","amount","of","data",".","seed",":","Random","seed","for","shuffling",".","serializers",":","The","serializers","used","to","serialize","and","deserialize","the","chunks",".","max_cache_size",":","The","maximum","cache","size","used","by","the","StreamingDataset","."]
code: def __init__(
      self,
      input_dir: Union[str, "RemoteDir"],
      item_loader: Optional[BaseItemLoader] = None,
      shuffle: bool = False,
      drop_last: bool = False,
      seed: int = 42,
      serializers: Optional[Dict[str, Serializer]] = None,
      max_cache_size: Union[int, str] = "100GB",
  ) -> None:
      """The streaming dataset can be used once your data have been optimised using the DatasetOptimiser class.
  
      Arguments:
          input_dir: Path to the folder where the input data is stored.
          item_loader: The logic to load an item from a chunk.
          shuffle: Whether to shuffle the data.
          drop_last: If `True`, drops the last items to ensure that
              all processes/workers return the same amount of data.
          seed: Random seed for shuffling.
          serializers: The serializers used to serialize and deserialize the chunks.
          max_cache_size: The maximum cache size used by the StreamingDataset.
  
      """
      super().__init__()
      if not isinstance(shuffle, bool):
          raise ValueError(
              f"Shuffle should be a boolean. Found {shuffle}"
          )
  
      if isinstance(input_dir, RemoteDir):
          input_dir = Dir(
              path=input_dir.cache_dir, url=input_dir.remote
          )
  
      input_dir = _resolve_dir(input_dir)
  
      self.input_dir = input_dir
      self.item_loader = item_loader
      self.shuffle: bool = shuffle
      self.drop_last = drop_last
      self.seed = seed
      self.max_cache_size = max_cache_size
  
      self.cache: Optional[Cache] = None
      self.distributed_env = _DistributedEnv.detect()
      self.worker_env: Optional[_WorkerEnv] = None
      self.worker_chunks: List[int] = []
      self.worker_intervals: List[List[int]] = []
      self.current_indexes: List[int] = []
      self.chunk_index = 0
      self.num_chunks: Optional[int] = None
      self.global_index = 0
      self.index = 0
      self.has_triggered_download = False
      self.min_items_per_replica: Optional[int] = None
      self.current_epoch = 1
      self.random_state = None
      self.shuffler: Optional[Shuffle] = None
      self.serializers = serializers
      self._state_dict: Optional[Dict[str, Any]] = None
  
imports: import hashlib
  import os
  from dataclasses import dataclass
  from time import time
  from typing import Any, Dict, List, Optional, Tuple, Union
  import numpy
  from torch.utils.data import IterableDataset, get_worker_info
  from lightning.data.streaming import Cache
  from lightning.data.streaming.constants import _DEFAULT_CACHE_DIR, _INDEX_FILENAME, _LIGHTNING_CLOUD_LATEST
  from lightning.data.streaming.item_loader import BaseItemLoader
  from lightning.data.streaming.sampler import ChunkedIndex
  from lightning.data.streaming.serializers import Serializer
  from lightning.data.streaming.shuffle import FullShuffle, NoShuffle, Shuffle
  from lightning.data.utilities.env import Environment, _DistributedEnv, _WorkerEnv
label_desc: Use image node_id 1 to create a new StreamingDataset object from inherited base classes: IterableDataset with example: obj = StreamingDataset(input_dir, item_loader, shuffle, drop_last, seed, serializers, max_cache_size)