class_docstr: This buffer adds replayed samples to a stream of new experiences.
  
  - Any newly added batch (`add()`) is immediately returned upon
  the next `sample` call (close to on-policy) as well as being moved
  into the buffer.
  - Additionally, a certain number of old samples is mixed into the
  returned sample according to a given "replay ratio".
  - If >1 calls to `add()` are made without any `sample()` calls
  in between, all newly added batches are returned (plus some older samples
  according to the "replay ratio").
  
  .. testcode::
      :skipif: True
  
      # replay ratio 0.66 (2/3 replayed, 1/3 new samples):
      buffer = MultiAgentMixInReplayBuffer(capacity=100,
                                           replay_ratio=0.66)
      buffer.add(<A>)
      buffer.add(<B>)
      buffer.sample(1)
  
  .. testoutput::
  
      ..[<A>, <B>, <B>]
  
  .. testcode::
      :skipif: True
  
      buffer.add(<C>)
      buffer.sample(1)
  
  .. testoutput::
  
      [<C>, <A>, <B>]
      or: [<C>, <A>, <A>], [<C>, <B>, <A>] or [<C>, <B>, <B>],
      but always <C> as it is the newest sample
  
  .. testcode::
      :skipif: True
  
      buffer.add(<D>)
      buffer.sample(1)
  
  .. testoutput::
  
      [<D>, <A>, <C>]
      or [<D>, <A>, <A>], [<D>, <B>, <A>] or [<D>, <B>, <C>], etc..
      but always <D> as it is the newest sample
  
  .. testcode::
      :skipif: True
  
      # replay proportion 0.0 -> replay disabled:
      buffer = MixInReplay(capacity=100, replay_ratio=0.0)
      buffer.add(<A>)
      buffer.sample()
  
  .. testoutput::
  
      [<A>]
  
  .. testcode::
      :skipif: True
  
      buffer.add(<B>)
      buffer.sample()
  
  .. testoutput::
  
      [<B>]
class_docstr_tok: ["This","buffer","adds","replayed","samples","to","a","stream","of","new","experiences",".","-","Any","newly","added","batch","(","`","add","(",")","`",")","is","immediately","returned","upon","the","next","`","sample","`","call","(","close","to","on-policy",")","as","well","as","being","moved","into","the","buffer",".","-","Additionally",",","a","certain","number","of","old","samples","is","mixed","into","the","returned","sample","according","to","a","given","``","replay","ratio","''",".","-","If",">","1","calls","to","`","add","(",")","`","are","made","without","any","`","sample","(",")","`","calls","in","between",",","all","newly","added","batches","are","returned","(","plus","some","older","samples","according","to","the","``","replay","ratio","''",")",".","..","testcode",":",":",":","skipif",":","True","#","replay","ratio","0.66","(","2\/3","replayed",",","1\/3","new","samples",")",":","buffer","=","MultiAgentMixInReplayBuffer","(","capacity=100",",","replay_ratio=0.66",")","buffer.add","(","<","A",">",")","buffer.add","(","<","B",">",")","buffer.sample","(","1",")","..","testoutput",":",":","..","[","<","A",">",",","<","B",">",",","<","B",">","]","..","testcode",":",":",":","skipif",":","True","buffer.add","(","<","C",">",")","buffer.sample","(","1",")","..","testoutput",":",":","[","<","C",">",",","<","A",">",",","<","B",">","]","or",":","[","<","C",">",",","<","A",">",",","<","A",">","]",",","[","<","C",">",",","<","B",">",",","<","A",">","]","or","[","<","C",">",",","<","B",">",",","<","B",">","]",",","but","always","<","C",">","as","it","is","the","newest","sample","..","testcode",":",":",":","skipif",":","True","buffer.add","(","<","D",">",")","buffer.sample","(","1",")","..","testoutput",":",":","[","<","D",">",",","<","A",">",",","<","C",">","]","or","[","<","D",">",",","<","A",">",",","<","A",">","]",",","[","<","D",">",",","<","B",">",",","<","A",">","]","or","[","<","D",">",",","<","B",">",",","<","C",">","]",",","etc","..","but","always","<","D",">","as","it","is","the","newest","sample","..","testcode",":",":",":","skipif",":","True","#","replay","proportion","0.0","-",">","replay","disabled",":","buffer","=","MixInReplay","(","capacity=100",",","replay_ratio=0.0",")","buffer.add","(","<","A",">",")","buffer.sample","(",")","..","testoutput",":",":","[","<","A",">","]","..","testcode",":",":",":","skipif",":","True","buffer.add","(","<","B",">",")","buffer.sample","(",")","..","testoutput",":",":","[","<","B",">","]"]
docstr: Initializes MultiAgentMixInReplayBuffer instance.
  
  Args:
      capacity: The capacity of the buffer, measured in `storage_unit`.
      storage_unit: Either 'timesteps', 'sequences' or
          'episodes'. Specifies how experiences are stored. If they
          are stored in episodes, replay_sequence_length is ignored.
      num_shards: The number of buffer shards that exist in total
          (including this one).
      replay_mode: One of "independent" or "lockstep". Determines,
          whether batches are sampled independently or to an equal
          amount.
      replay_sequence_override: If True, ignore sequences found in incoming
          batches, slicing them into sequences as specified by
          `replay_sequence_length` and `replay_sequence_burn_in`. This only has
          an effect if storage_unit is `sequences`.
      replay_sequence_length: The sequence length (T) of a single
          sample. If > 1, we will sample B x T from this buffer. This
          only has an effect if storage_unit is 'timesteps'.
      replay_burn_in: The burn-in length in case
          `replay_sequence_length` > 0. This is the number of timesteps
          each sequence overlaps with the previous one to generate a
          better internal state (=state after the burn-in), instead of
          starting from 0.0 each RNN rollout.
      replay_zero_init_states: Whether the initial states in the
          buffer (if replay_sequence_length > 0) are alwayas 0.0 or
          should be updated with the previous train_batch state outputs.
      replay_ratio: Ratio of replayed samples in the returned
          batches. E.g. a ratio of 0.0 means only return new samples
          (no replay), a ratio of 0.5 means always return newest sample
          plus one old one (1:1), a ratio of 0.66 means always return
          the newest sample plus 2 old (replayed) ones (1:2), etc...
      underlying_buffer_config: A config that contains all necessary
          constructor arguments and arguments for methods to call on
          the underlying buffers. This replaces the standard behaviour
          of the underlying PrioritizedReplayBuffer. The config
          follows the conventions of the general
          replay_buffer_config. kwargs for subsequent calls of methods
          may also be included. Example:
          "replay_buffer_config": {"type": PrioritizedReplayBuffer,
          "capacity": 10, "storage_unit": "timesteps",
          prioritized_replay_alpha: 0.5, prioritized_replay_beta: 0.5,
          prioritized_replay_eps: 0.5}
      prioritized_replay_alpha: Alpha parameter for a prioritized
          replay buffer. Use 0.0 for no prioritization.
      prioritized_replay_beta: Beta parameter for a prioritized
          replay buffer.
      prioritized_replay_eps: Epsilon parameter for a prioritized
          replay buffer.
      **kwargs: Forward compatibility kwargs.
docstr_tok: ["Initializes","MultiAgentMixInReplayBuffer","instance",".","Args",":","capacity",":","The","capacity","of","the","buffer",",","measured","in","`","storage_unit","`",".","storage_unit",":","Either","'timesteps","'",",","'sequences","'","or","'episodes","'",".","Specifies","how","experiences","are","stored",".","If","they","are","stored","in","episodes",",","replay_sequence_length","is","ignored",".","num_shards",":","The","number","of","buffer","shards","that","exist","in","total","(","including","this","one",")",".","replay_mode",":","One","of","``","independent","''","or","``","lockstep","''",".","Determines",",","whether","batches","are","sampled","independently","or","to","an","equal","amount",".","replay_sequence_override",":","If","True",",","ignore","sequences","found","in","incoming","batches",",","slicing","them","into","sequences","as","specified","by","`","replay_sequence_length","`","and","`","replay_sequence_burn_in","`",".","This","only","has","an","effect","if","storage_unit","is","`","sequences","`",".","replay_sequence_length",":","The","sequence","length","(","T",")","of","a","single","sample",".","If",">","1",",","we","will","sample","B","x","T","from","this","buffer",".","This","only","has","an","effect","if","storage_unit","is","'timesteps","'",".","replay_burn_in",":","The","burn-in","length","in","case","`","replay_sequence_length","`",">","0",".","This","is","the","number","of","timesteps","each","sequence","overlaps","with","the","previous","one","to","generate","a","better","internal","state","(","=state","after","the","burn-in",")",",","instead","of","starting","from","0.0","each","RNN","rollout",".","replay_zero_init_states",":","Whether","the","initial","states","in","the","buffer","(","if","replay_sequence_length",">","0",")","are","alwayas","0.0","or","should","be","updated","with","the","previous","train_batch","state","outputs",".","replay_ratio",":","Ratio","of","replayed","samples","in","the","returned","batches",".","E.g",".","a","ratio","of","0.0","means","only","return","new","samples","(","no","replay",")",",","a","ratio","of","0.5","means","always","return","newest","sample","plus","one","old","one","(","1:1",")",",","a","ratio","of","0.66","means","always","return","the","newest","sample","plus","2","old","(","replayed",")","ones","(","1:2",")",",","etc","...","underlying_buffer_config",":","A","config","that","contains","all","necessary","constructor","arguments","and","arguments","for","methods","to","call","on","the","underlying","buffers",".","This","replaces","the","standard","behaviour","of","the","underlying","PrioritizedReplayBuffer",".","The","config","follows","the","conventions","of","the","general","replay_buffer_config",".","kwargs","for","subsequent","calls","of","methods","may","also","be","included",".","Example",":","``","replay_buffer_config","''",":","{","``","type","''",":","PrioritizedReplayBuffer",",","``","capacity","''",":","10",",","``","storage_unit","''",":","``","timesteps","''",",","prioritized_replay_alpha",":","0.5",",","prioritized_replay_beta",":","0.5",",","prioritized_replay_eps",":","0.5","}","prioritized_replay_alpha",":","Alpha","parameter","for","a","prioritized","replay","buffer",".","Use","0.0","for","no","prioritization",".","prioritized_replay_beta",":","Beta","parameter","for","a","prioritized","replay","buffer",".","prioritized_replay_eps",":","Epsilon","parameter","for","a","prioritized","replay","buffer",".","*","*","kwargs",":","Forward","compatibility","kwargs","."]
code: def __init__(
      self,
      capacity: int = 10000,
      storage_unit: str = "timesteps",
      num_shards: int = 1,
      replay_mode: str = "independent",
      replay_sequence_override: bool = True,
      replay_sequence_length: int = 1,
      replay_burn_in: int = 0,
      replay_zero_init_states: bool = True,
      replay_ratio: float = 0.66,
      underlying_buffer_config: dict = None,
      prioritized_replay_alpha: float = 0.6,
      prioritized_replay_beta: float = 0.4,
      prioritized_replay_eps: float = 1e-6,
      **kwargs,
  ):
      """Initializes MultiAgentMixInReplayBuffer instance.
  
      Args:
          capacity: The capacity of the buffer, measured in `storage_unit`.
          storage_unit: Either 'timesteps', 'sequences' or
              'episodes'. Specifies how experiences are stored. If they
              are stored in episodes, replay_sequence_length is ignored.
          num_shards: The number of buffer shards that exist in total
              (including this one).
          replay_mode: One of "independent" or "lockstep". Determines,
              whether batches are sampled independently or to an equal
              amount.
          replay_sequence_override: If True, ignore sequences found in incoming
              batches, slicing them into sequences as specified by
              `replay_sequence_length` and `replay_sequence_burn_in`. This only has
              an effect if storage_unit is `sequences`.
          replay_sequence_length: The sequence length (T) of a single
              sample. If > 1, we will sample B x T from this buffer. This
              only has an effect if storage_unit is 'timesteps'.
          replay_burn_in: The burn-in length in case
              `replay_sequence_length` > 0. This is the number of timesteps
              each sequence overlaps with the previous one to generate a
              better internal state (=state after the burn-in), instead of
              starting from 0.0 each RNN rollout.
          replay_zero_init_states: Whether the initial states in the
              buffer (if replay_sequence_length > 0) are alwayas 0.0 or
              should be updated with the previous train_batch state outputs.
          replay_ratio: Ratio of replayed samples in the returned
              batches. E.g. a ratio of 0.0 means only return new samples
              (no replay), a ratio of 0.5 means always return newest sample
              plus one old one (1:1), a ratio of 0.66 means always return
              the newest sample plus 2 old (replayed) ones (1:2), etc...
          underlying_buffer_config: A config that contains all necessary
              constructor arguments and arguments for methods to call on
              the underlying buffers. This replaces the standard behaviour
              of the underlying PrioritizedReplayBuffer. The config
              follows the conventions of the general
              replay_buffer_config. kwargs for subsequent calls of methods
              may also be included. Example:
              "replay_buffer_config": {"type": PrioritizedReplayBuffer,
              "capacity": 10, "storage_unit": "timesteps",
              prioritized_replay_alpha: 0.5, prioritized_replay_beta: 0.5,
              prioritized_replay_eps: 0.5}
          prioritized_replay_alpha: Alpha parameter for a prioritized
              replay buffer. Use 0.0 for no prioritization.
          prioritized_replay_beta: Beta parameter for a prioritized
              replay buffer.
          prioritized_replay_eps: Epsilon parameter for a prioritized
              replay buffer.
          **kwargs: Forward compatibility kwargs.
      """
      if not 0 <= replay_ratio <= 1:
          raise ValueError("Replay ratio must be within [0, 1]")
  
      MultiAgentPrioritizedReplayBuffer.__init__(
          self,
          capacity=capacity,
          storage_unit=storage_unit,
          num_shards=num_shards,
          replay_mode=replay_mode,
          replay_sequence_override=replay_sequence_override,
          replay_sequence_length=replay_sequence_length,
          replay_burn_in=replay_burn_in,
          replay_zero_init_states=replay_zero_init_states,
          underlying_buffer_config=underlying_buffer_config,
          prioritized_replay_alpha=prioritized_replay_alpha,
          prioritized_replay_beta=prioritized_replay_beta,
          prioritized_replay_eps=prioritized_replay_eps,
          **kwargs,
      )
  
      self.replay_ratio = replay_ratio
  
      self.last_added_batches = collections.defaultdict(list)
  
imports: import collections
  import logging
  import random
  from typing import Any, Dict, Optional
  import numpy
  from ray.rllib.policy.rnn_sequencing import timeslice_along_seq_lens_with_overlap
  from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID, SampleBatch, concat_samples_into_ma_batch
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.replay_buffers.multi_agent_prioritized_replay_buffer import MultiAgentPrioritizedReplayBuffer
  from ray.rllib.utils.replay_buffers.multi_agent_replay_buffer import MultiAgentReplayBuffer, ReplayMode, merge_dicts_with_warning
  from ray.rllib.utils.replay_buffers.replay_buffer import _ALL_POLICIES, StorageUnit
  from ray.rllib.utils.typing import PolicyID, SampleBatchType
  from ray.util.annotations import DeveloperAPI
  from ray.util.debug import log_once
label_desc: Use image node_id 1 to create a new MultiAgentMixInReplayBuffer object from inherited base classes: MultiAgentPrioritizedReplayBuffer with example: obj = MultiAgentMixInReplayBuffer(capacity, storage_unit, num_shards, replay_mode, replay_sequence_override, replay_sequence_length, replay_burn_in, replay_zero_init_states, replay_ratio, underlying_buffer_config, prioritized_replay_alpha, prioritized_replay_beta, prioritized_replay_eps)