code: import tensorflow as tf
  import numpy as np
  import gym
  import time
  
  
  np.random.seed(1)
  tf.set_random_seed(1)
  
  #####################  hyper parameters  ####################
  
  # MAX_EPISODES = 200
  # MAX_EP_STEPS = 200
  # LR_A = 0.001    # learning rate for actor
  # LR_C = 0.001    # learning rate for critic
  # GAMMA = 0.9     # reward discount
  REPLACEMENT = [
      dict(name='soft', tau=0.01),
      dict(name='hard', rep_iter_a=600, rep_iter_c=500)
  ][0]            # you can try different target replacement strategies
  MEMORY_CAPACITY = 10000
  
  BATCH_SIZE = 32
  RENDER = False
  OUTPUT_GRAPH = False
  ENV_NAME = 'Pendulum-v0'
  
  ###############################  DDPG  #####################################
  class DDPG(object):
      def __init__(self, a_dim, s_dim, a_bound, LR_A=0.001, LR_C=0.001, GAMMA=0.9 ,TAU=0.01):
          self.memory = np.zeros((MEMORY_CAPACITY, s_dim * 2 + a_dim + 1), dtype=np.float32)
          self.pointer = 0
          self.sess = tf.Session()
  
          self.a_dim, self.s_dim, self.a_bound = a_dim, s_dim, a_bound,
          self.S = tf.placeholder(tf.float32, [None, s_dim], 's')
          self.S_ = tf.placeholder(tf.float32, [None, s_dim], 's_')
          self.R = tf.placeholder(tf.float32, [None, 1], 'r')
          self.MEMORY_CAPACITY = MEMORY_CAPACITY
          self.var = 0.25
  
          self.a = self._build_a(self.S, )
          q = self._build_c(self.S, self.a, )
          a_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Actor')
          c_params = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='Critic')
          ema = tf.train.ExponentialMovingAverage(decay=1 - TAU)  # soft replacement
  
          def ema_getter(getter, name, *args, **kwargs):
              return ema.average(getter(name, *args, **kwargs))
  
          target_update = [ema.apply(a_params), ema.apply(c_params)]  # soft update operation
          a_ = self._build_a(self.S_, reuse=True, custom_getter=ema_getter)  # replaced target parameters
          q_ = self._build_c(self.S_, a_, reuse=True, custom_getter=ema_getter)
  
          a_loss = - tf.reduce_mean(q)  # maximize the q
          self.atrain = tf.train.AdamOptimizer(LR_A).minimize(a_loss, var_list=a_params)
  
          with tf.control_dependencies(target_update):  # soft replacement happened at here
              q_target = self.R + GAMMA * q_
              td_error = tf.losses.mean_squared_error(labels=q_target, predictions=q)
              self.ctrain = tf.train.AdamOptimizer(LR_C).minimize(td_error, var_list=c_params)
  
          self.sess.run(tf.global_variables_initializer())
  
      def choose_action(self, s):
  
          action_prob = self.sess.run(self.a, {self.S: s})[0]
          action_prob = np.clip(np.random.normal(action_prob, self.var), 0, 1)
          p = np.array([1-action_prob,action_prob])
          print(p)
          print("Var:",self.var)
          action = np.random.choice(range(2), p=p.ravel())
  
          return action
  
      def learn(self):
          indices = np.random.choice(MEMORY_CAPACITY, size=BATCH_SIZE)
          bt = self.memory[indices, :]
          bs = bt[:, :self.s_dim]
          ba = bt[:, self.s_dim: self.s_dim + self.a_dim]
          br = bt[:, -self.s_dim - 1: -self.s_dim]
          bs_ = bt[:, -self.s_dim:]
  
          self.var *= .9995
  
          self.sess.run(self.atrain, {self.S: bs})
          self.sess.run(self.ctrain, {self.S: bs, self.a: ba, self.R: br, self.S_: bs_})
  
      def store_transition(self, s, a, r, s_):
          transition = np.hstack((s, a, [r], s_))
          index = self.pointer % MEMORY_CAPACITY  # replace the old memory with new memory
          self.memory[index, :] = transition
          self.pointer += 1
  
      def _build_a(self, s, reuse=None, custom_getter=None):
          trainable = True if reuse is None else False
          with tf.variable_scope('Actor', reuse=reuse, custom_getter=custom_getter):
              l1 = tf.layers.dense(s, 30, activation=tf.nn.relu, name='l1', trainable=trainable)
              # l2 = tf.layers.dense(l1, self.a_dim, activation=tf.nn.relu, name='a', trainable=trainable)
              l2 = tf.layers.dense(l1, 15, activation=tf.nn.relu, name='a', trainable=trainable)
              a = tf.layers.dense(
                  inputs=l2,
                  units=2,  # output units
                  activation=tf.nn.softmax,  # get action probabilities
                  kernel_initializer=tf.random_normal_initializer(0., .1),  # weights
                  bias_initializer=tf.constant_initializer(0.1),  # biases
                  name='acts_prob'
              )
              # print(tf.multiply(l2, self.a_bound, name='scaled_a'))
              # print(np.array(a[:,1:2]).shape)
              return a[:,1:2]
  
      def _build_c(self, s, a, reuse=None, custom_getter=None):
          trainable = True if reuse is None else False
          with tf.variable_scope('Critic', reuse=reuse, custom_getter=custom_getter):
              n_l1 = 30
              w1_s = tf.get_variable('w1_s', [self.s_dim, n_l1], trainable=trainable)
              w1_a = tf.get_variable('w1_a', [self.a_dim, n_l1], trainable=trainable)
              b1 = tf.get_variable('b1', [1, n_l1], trainable=trainable)
              net = tf.nn.relu(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
              return tf.layers.dense(net, 1, trainable=trainable)  # Q(s,a)
apis: tensorflow.get_variable tensorflow.control_dependencies tensorflow.train.ExponentialMovingAverage tensorflow.train.AdamOptimizer numpy.hstack tensorflow.get_collection tensorflow.layers.dense tensorflow.Session tensorflow.random_normal_initializer numpy.zeros tensorflow.matmul numpy.random.choice tensorflow.placeholder tensorflow.global_variables_initializer tensorflow.set_random_seed numpy.array tensorflow.losses.mean_squared_error numpy.random.seed tensorflow.reduce_mean tensorflow.constant_initializer numpy.random.normal tensorflow.variable_scope
api_extract: [(7, 'numpy.random.seed', 'np.random.seed', (['(1)'], {}), True, 'import numpy as np\n'), (8, 'tensorflow.set_random_seed', 'tf.set_random_seed', (['(1)'], {}), True, 'import tensorflow as tf\n'), (31, 'numpy.zeros', 'np.zeros', (['(MEMORY_CAPACITY, s_dim * 2 + a_dim + 1)'], {'dtype': 'np.float32'}), True, 'import numpy as np\n'), (33, 'tensorflow.Session', 'tf.Session', ([], {}), True, 'import tensorflow as tf\n'), (36, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[None, s_dim]', '"""s"""'], {}), True, 'import tensorflow as tf\n'), (37, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[None, s_dim]', '"""s_"""'], {}), True, 'import tensorflow as tf\n'), (38, 'tensorflow.placeholder', 'tf.placeholder', (['tf.float32', '[None, 1]', '"""r"""'], {}), True, 'import tensorflow as tf\n'), (44, 'tensorflow.get_collection', 'tf.get_collection', (['tf.GraphKeys.TRAINABLE_VARIABLES'], {'scope': '"""Actor"""'}), True, 'import tensorflow as tf\n'), (45, 'tensorflow.get_collection', 'tf.get_collection', (['tf.GraphKeys.TRAINABLE_VARIABLES'], {'scope': '"""Critic"""'}), True, 'import tensorflow as tf\n'), (46, 'tensorflow.train.ExponentialMovingAverage', 'tf.train.ExponentialMovingAverage', ([], {'decay': '(1 - TAU)'}), True, 'import tensorflow as tf\n'), (69, 'numpy.array', 'np.array', (['[1 - action_prob, action_prob]'], {}), True, 'import numpy as np\n'), (77, 'numpy.random.choice', 'np.random.choice', (['MEMORY_CAPACITY'], {'size': 'BATCH_SIZE'}), True, 'import numpy as np\n'), (90, 'numpy.hstack', 'np.hstack', (['(s, a, [r], s_)'], {}), True, 'import numpy as np\n'), (55, 'tensorflow.reduce_mean', 'tf.reduce_mean', (['q'], {}), True, 'import tensorflow as tf\n'), (58, 'tensorflow.control_dependencies', 'tf.control_dependencies', (['target_update'], {}), True, 'import tensorflow as tf\n'), (60, 'tensorflow.losses.mean_squared_error', 'tf.losses.mean_squared_error', ([], {'labels': 'q_target', 'predictions': 'q'}), True, 'import tensorflow as tf\n'), (63, 'tensorflow.global_variables_initializer', 'tf.global_variables_initializer', ([], {}), True, 'import tensorflow as tf\n'), (68, 'numpy.random.normal', 'np.random.normal', (['action_prob', 'self.var'], {}), True, 'import numpy as np\n'), (97, 'tensorflow.variable_scope', 'tf.variable_scope', (['"""Actor"""'], {'reuse': 'reuse', 'custom_getter': 'custom_getter'}), True, 'import tensorflow as tf\n'), (98, 'tensorflow.layers.dense', 'tf.layers.dense', (['s', '(30)'], {'activation': 'tf.nn.relu', 'name': '"""l1"""', 'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (100, 'tensorflow.layers.dense', 'tf.layers.dense', (['l1', '(15)'], {'activation': 'tf.nn.relu', 'name': '"""a"""', 'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (115, 'tensorflow.variable_scope', 'tf.variable_scope', (['"""Critic"""'], {'reuse': 'reuse', 'custom_getter': 'custom_getter'}), True, 'import tensorflow as tf\n'), (117, 'tensorflow.get_variable', 'tf.get_variable', (['"""w1_s"""', '[self.s_dim, n_l1]'], {'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (118, 'tensorflow.get_variable', 'tf.get_variable', (['"""w1_a"""', '[self.a_dim, n_l1]'], {'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (119, 'tensorflow.get_variable', 'tf.get_variable', (['"""b1"""', '[1, n_l1]'], {'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (121, 'tensorflow.layers.dense', 'tf.layers.dense', (['net', '(1)'], {'trainable': 'trainable'}), True, 'import tensorflow as tf\n'), (56, 'tensorflow.train.AdamOptimizer', 'tf.train.AdamOptimizer', (['LR_A'], {}), True, 'import tensorflow as tf\n'), (61, 'tensorflow.train.AdamOptimizer', 'tf.train.AdamOptimizer', (['LR_C'], {}), True, 'import tensorflow as tf\n'), (105, 'tensorflow.random_normal_initializer', 'tf.random_normal_initializer', (['(0.0)', '(0.1)'], {}), True, 'import tensorflow as tf\n'), (106, 'tensorflow.constant_initializer', 'tf.constant_initializer', (['(0.1)'], {}), True, 'import tensorflow as tf\n'), (120, 'tensorflow.matmul', 'tf.matmul', (['s', 'w1_s'], {}), True, 'import tensorflow as tf\n'), (120, 'tensorflow.matmul', 'tf.matmul', (['a', 'w1_a'], {}), True, 'import tensorflow as tf\n')]