class_docstr: A replay buffer shard for multiagent setups.
  
  This buffer is meant to be run in parallel to distribute experiences
  across `num_shards` shards. Unlike simpler buffers, it holds a set of
  buffers - one for each policy ID.
docstr: Initializes a MultiAgentReplayBuffer instance.
  
  Args:
      capacity: The capacity of the buffer, measured in `storage_unit`.
      storage_unit: Either 'timesteps', 'sequences' or
          'episodes'. Specifies how experiences are stored. If they
          are stored in episodes, replay_sequence_length is ignored.
      num_shards: The number of buffer shards that exist in total
          (including this one).
      replay_mode: One of "independent" or "lockstep". Determines,
          whether batches are sampled independently or to an equal
          amount.
      replay_sequence_override: If True, ignore sequences found in incoming
          batches, slicing them into sequences as specified by
          `replay_sequence_length` and `replay_sequence_burn_in`. This only has
          an effect if storage_unit is `sequences`.
      replay_sequence_length: The sequence length (T) of a single
          sample. If > 1, we will sample B x T from this buffer. This
          only has an effect if storage_unit is 'timesteps'.
      replay_burn_in: This is the number of timesteps
          each sequence overlaps with the previous one to generate a
          better internal state (=state after the burn-in), instead of
          starting from 0.0 each RNN rollout. This only has an effect
          if storage_unit is `sequences`.
      replay_zero_init_states: Whether the initial states in the
          buffer (if replay_sequence_length > 0) are alwayas 0.0 or
          should be updated with the previous train_batch state outputs.
      underlying_buffer_config: A config that contains all necessary
          constructor arguments and arguments for methods to call on
          the underlying buffers.
      ``**kwargs``: Forward compatibility kwargs.
docstr_tok: ["Initializes","a","MultiAgentReplayBuffer","instance",".","Args",":","capacity",":","The","capacity","of","the","buffer",",","measured","in","`","storage_unit","`",".","storage_unit",":","Either","'timesteps","'",",","'sequences","'","or","'episodes","'",".","Specifies","how","experiences","are","stored",".","If","they","are","stored","in","episodes",",","replay_sequence_length","is","ignored",".","num_shards",":","The","number","of","buffer","shards","that","exist","in","total","(","including","this","one",")",".","replay_mode",":","One","of","``","independent","''","or","``","lockstep","''",".","Determines",",","whether","batches","are","sampled","independently","or","to","an","equal","amount",".","replay_sequence_override",":","If","True",",","ignore","sequences","found","in","incoming","batches",",","slicing","them","into","sequences","as","specified","by","`","replay_sequence_length","`","and","`","replay_sequence_burn_in","`",".","This","only","has","an","effect","if","storage_unit","is","`","sequences","`",".","replay_sequence_length",":","The","sequence","length","(","T",")","of","a","single","sample",".","If",">","1",",","we","will","sample","B","x","T","from","this","buffer",".","This","only","has","an","effect","if","storage_unit","is","'timesteps","'",".","replay_burn_in",":","This","is","the","number","of","timesteps","each","sequence","overlaps","with","the","previous","one","to","generate","a","better","internal","state","(","=state","after","the","burn-in",")",",","instead","of","starting","from","0.0","each","RNN","rollout",".","This","only","has","an","effect","if","storage_unit","is","`","sequences","`",".","replay_zero_init_states",":","Whether","the","initial","states","in","the","buffer","(","if","replay_sequence_length",">","0",")","are","alwayas","0.0","or","should","be","updated","with","the","previous","train_batch","state","outputs",".","underlying_buffer_config",":","A","config","that","contains","all","necessary","constructor","arguments","and","arguments","for","methods","to","call","on","the","underlying","buffers",".","``","*","*","kwargs","``",":","Forward","compatibility","kwargs","."]
code: def __init__(
      self,
      capacity: int = 10000,
      storage_unit: str = "timesteps",
      num_shards: int = 1,
      replay_mode: str = "independent",
      replay_sequence_override: bool = True,
      replay_sequence_length: int = 1,
      replay_burn_in: int = 0,
      replay_zero_init_states: bool = True,
      underlying_buffer_config: dict = None,
      **kwargs,
  ):
      """Initializes a MultiAgentReplayBuffer instance.
  
      Args:
          capacity: The capacity of the buffer, measured in `storage_unit`.
          storage_unit: Either 'timesteps', 'sequences' or
              'episodes'. Specifies how experiences are stored. If they
              are stored in episodes, replay_sequence_length is ignored.
          num_shards: The number of buffer shards that exist in total
              (including this one).
          replay_mode: One of "independent" or "lockstep". Determines,
              whether batches are sampled independently or to an equal
              amount.
          replay_sequence_override: If True, ignore sequences found in incoming
              batches, slicing them into sequences as specified by
              `replay_sequence_length` and `replay_sequence_burn_in`. This only has
              an effect if storage_unit is `sequences`.
          replay_sequence_length: The sequence length (T) of a single
              sample. If > 1, we will sample B x T from this buffer. This
              only has an effect if storage_unit is 'timesteps'.
          replay_burn_in: This is the number of timesteps
              each sequence overlaps with the previous one to generate a
              better internal state (=state after the burn-in), instead of
              starting from 0.0 each RNN rollout. This only has an effect
              if storage_unit is `sequences`.
          replay_zero_init_states: Whether the initial states in the
              buffer (if replay_sequence_length > 0) are alwayas 0.0 or
              should be updated with the previous train_batch state outputs.
          underlying_buffer_config: A config that contains all necessary
              constructor arguments and arguments for methods to call on
              the underlying buffers.
          ``**kwargs``: Forward compatibility kwargs.
      """
      shard_capacity = capacity // num_shards
      ReplayBuffer.__init__(self, capacity, storage_unit)
  
      # If the user provides an underlying buffer config, we use to
      # instantiate and interact with underlying buffers
      self.underlying_buffer_config = underlying_buffer_config
      if self.underlying_buffer_config is not None:
          self.underlying_buffer_call_args = (
              self.underlying_buffer_config
          )
      else:
          self.underlying_buffer_call_args = {}
      self.replay_sequence_override = replay_sequence_override
      self.replay_mode = replay_mode
      self.replay_sequence_length = replay_sequence_length
      self.replay_burn_in = replay_burn_in
      self.replay_zero_init_states = replay_zero_init_states
      self.replay_sequence_override = replay_sequence_override
  
      if (
          replay_sequence_length > 1
          and self.storage_unit is not StorageUnit.SEQUENCES
      ):
          logger.warning(
              "MultiAgentReplayBuffer configured with "
              "`replay_sequence_length={}`, but `storage_unit={}`. "
              "replay_sequence_length will be ignored and set to 1.".format(
                  replay_sequence_length, storage_unit
              )
          )
          self.replay_sequence_length = 1
  
      if (
          replay_sequence_length == 1
          and self.storage_unit is StorageUnit.SEQUENCES
      ):
          logger.warning(
              "MultiAgentReplayBuffer configured with "
              "`replay_sequence_length={}`, but `storage_unit={}`. "
              "This will result in sequences equal to timesteps.".format(
                  replay_sequence_length, storage_unit
              )
          )
  
      if replay_mode in ["lockstep", ReplayMode.LOCKSTEP]:
          self.replay_mode = ReplayMode.LOCKSTEP
          if self.storage_unit in [
              StorageUnit.EPISODES,
              StorageUnit.SEQUENCES,
          ]:
              raise ValueError(
                  "MultiAgentReplayBuffer does not support "
                  "lockstep mode with storage unit `episodes`"
                  "or `sequences`."
              )
      elif replay_mode in ["independent", ReplayMode.INDEPENDENT]:
          self.replay_mode = ReplayMode.INDEPENDENT
      else:
          raise ValueError(
              "Unsupported replay mode: {}".format(replay_mode)
          )
  
      if self.underlying_buffer_config:
          ctor_args = {
              **{
                  "capacity": shard_capacity,
                  "storage_unit": StorageUnit.FRAGMENTS,
              },
              **self.underlying_buffer_config,
          }
  
          def new_buffer():
              return from_config(
                  self.underlying_buffer_config["type"], ctor_args
              )
  
      else:
          # Default case
          def new_buffer():
              self.underlying_buffer_call_args = {}
              return ReplayBuffer(
                  self.capacity,
                  storage_unit=StorageUnit.FRAGMENTS,
              )
  
      self.replay_buffers = collections.defaultdict(new_buffer)
  
      # Metrics.
      self.add_batch_timer = _Timer()
      self.replay_timer = _Timer()
      self._num_added = 0
  
imports: import collections
  import logging
  from enum import Enum
  from typing import Any, Dict, Optional
  from ray.util.timer import _Timer
  from ray.rllib.policy.rnn_sequencing import timeslice_along_seq_lens_with_overlap
  from ray.rllib.policy.sample_batch import MultiAgentBatch, SampleBatch
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import Deprecated
  from ray.rllib.utils.from_config import from_config
  from ray.rllib.utils.replay_buffers.replay_buffer import _ALL_POLICIES, ReplayBuffer, StorageUnit
  from ray.rllib.utils.typing import PolicyID, SampleBatchType
  from ray.util.annotations import DeveloperAPI
  from ray.util.debug import log_once
label_desc: Use image node_id 1 to create a new MultiAgentReplayBuffer object from inherited base classes: ReplayBuffer with example: obj = MultiAgentReplayBuffer(capacity, storage_unit, num_shards, replay_mode, replay_sequence_override, replay_sequence_length, replay_burn_in, replay_zero_init_states, underlying_buffer_config)