text: import random
  
  import numpy as np
  import tensorflow as tf
  
  
  class ReplayBuffer(object):
  
      def __init__(self, size):
          self._storage = []
          self._maxsize = size
          self._next_idx = 0
  
      def __len__(self):
          return len(self._storage)
  
      def add(self, *args):
          if self._next_idx >= len(self._storage):
              self._storage.append(args)
          else:
              self._storage[self._next_idx] = args
          self._next_idx = (self._next_idx + 1) % self._maxsize
  
      def _encode_sample(self, idxes):
          b_o, b_a, b_r, b_o_, b_d = [], [], [], [], []
          for i in idxes:
              o, a, r, o_, d = self._storage[i]
              b_o.append(o)
              b_a.append(a)
              b_r.append(r)
              b_o_.append(o_)
              b_d.append(d)
          return (
              np.stack(b_o).astype('float32'),
              np.stack(b_a).astype('int32'),
              np.stack(b_r).astype('float32'),
              np.stack(b_o_).astype('float32'),
              np.stack(b_d).astype('float32'),
          )
  
      def sample(self, batch_size):
          indexes = range(len(self._storage))
          idxes = [random.choice(indexes) for _ in range(batch_size)]
          return self._encode_sample(idxes)
  
  
  def huber_loss(x):
      """Loss function for value"""
      return tf.where(tf.abs(x) < 1, tf.square(x) * 0.5, tf.abs(x) - 0.5)
  
  
  def sync(net, net_tar):
      """Copy q network to target q network"""
      for var, var_tar in zip(net.trainable_weights, net_tar.trainable_weights):
          var_tar.assign(var)
  