class_docstr: Open dataset in readonly mode.
  
  `DistributeDataset` is used to read datasets in a distributed manner.
  Data in this dataset will be distributed evenly in blocks to each worker in the `distributed communicator`.
  
  **Note** When all data has been read, reading dataset again will revert back to the first data.
  
  Args:
      path (str): Path to dataset.
      rank (int): Rank in distributed communicator. See: bmtrain.rank()
      world_size (int): Total workers in distributed communicator. See: bmtrain.world_size()
      block_size (int): Size of each block in bytes. All files in the same dataset should have the same block size. Default: 16MB
  
  Example:
      >>> dataset = DistributedDataset("/path/to/dataset")
      >>> for i in range(10):
      >>>     dataset.read()
class_docstr_tok: ["Open","dataset","in","readonly","mode",".","`","DistributeDataset","`","is","used","to","read","datasets","in","a","distributed","manner",".","Data","in","this","dataset","will","be","distributed","evenly","in","blocks","to","each","worker","in","the","`","distributed","communicator","`",".","*","*","Note","*","*","When","all","data","has","been","read",",","reading","dataset","again","will","revert","back","to","the","first","data",".","Args",":","path","(","str",")",":","Path","to","dataset",".","rank","(","int",")",":","Rank","in","distributed","communicator",".","See",":","bmtrain.rank","(",")","world_size","(","int",")",":","Total","workers","in","distributed","communicator",".","See",":","bmtrain.world_size","(",")","block_size","(","int",")",":","Size","of","each","block","in","bytes",".","All","files","in","the","same","dataset","should","have","the","same","block","size",".","Default",":","16MB","Example",":",">",">",">","dataset","=","DistributedDataset","(","``","\/path\/to\/dataset","''",")",">",">",">","for","i","in","range","(","10",")",":",">",">",">","dataset.read","(",")"]
docstr: Load dataset state.
  
  Args:
      state (dict): dataset state dict.
      strict (bool): If `strict` is True, world size needs to be the same as when exported.
  
  Example:
      >>> state = dataset.state_dict()
      >>>
code: def load_state_dict(self, state, strict: bool = True):
      """Load dataset state.
  
      Args:
          state (dict): dataset state dict.
          strict (bool): If `strict` is True, world size needs to be the same as when exported.
  
      Example:
          >>> state = dataset.state_dict()
          >>>
      """
      block_states: torch.LongTensor = state["states"]
      block_info: torch.LongTensor = state["block"]
  
      if block_states.size(0) != self._world_size:
          if strict:
              raise ValueError(
                  "world_size changed (%d -> %d)"
                  % (state["block"].size(0), self._world_size)
              )
          else:
              self._curr_block = None
              self._fp = None
              self._curr_fname = None
              self._repeat_times = int(block_info[0, 3].item())
  
              # re-shuffle unused blocks
              nw_unused_block: List[int] = []
              for i in range(block_states.size(0)):
                  # filter blocks that are not in this rank
                  num_unused_blocks: int = int(block_info[i, 2].item())
                  nw_unused_block.extend(
                      [
                          block_id
                          for block_id in block_states[
                              i, :num_unused_blocks
                          ].tolist()
                          if block_id % self._world_size == self._rank
                      ]
                  )
              if self._shuffle:
                  random.shuffle(nw_unused_block)
              self._unused_block = nw_unused_block
      else:
          (
              curr_block,
              inblock_offset,
              num_unused_blocks,
              self._repeat_times,
          ) = block_info[self._rank].tolist()
  
          if curr_block == -1:
              self._curr_block = None
          else:
              while True:
                  try:
                      self._curr_block = curr_block
                      f_info = self._get_block_file(self._curr_block)
                      self._open_file(
                          f_info.file_name,
                          (self._curr_block - f_info.block_begin)
                          * f_info.block_size
                          + inblock_offset,
                      )
                      self._unused_block = block_states[
                          self._rank, :num_unused_blocks
                      ].tolist()
                      break
                  except Exception:
                      print("Error: reading block!")
                      time.sleep(10)
      # end
      self._update_states()
  
imports: import io
  import os
  import struct
  from typing import List, Optional, Set
  import torch
  import bisect
  import bmtrain
  import json
  from .serializer import Serializer, PickleSerializer
  import random
  import string
  import time
label_desc: Use image node_id 9 for calling the DistributedDataset obj's underlying member method code with example usage: obj.load_state_dict(state, strict) without return types