docstr: Test Hindsight Experience Replay.
docstr_tok: ["Test","Hindsight","Experience","Replay","."]
code: def test_hindsight(self):
      """Test Hindsight Experience Replay."""
  
      # The environment is a plane in which the agent moves by steps until it reaches a randomly
      # positioned goal.  No reward is given until it reaches the goal.  That makes it very hard
      # to learn by standard methods, since it may take a very long time to receive any feedback
      # at all.  Using hindsight makes it much easier.
  
      class TestEnvironment(dc.rl.Environment):
          def __init__(self):
              super(TestEnvironment, self).__init__((4,), 4)
              self.moves = [(-1, 0), (1, 0), (0, -1), (0, 1)]
  
          def reset(self):
              self._state = np.concatenate(
                  [[0, 0], np.random.randint(-50, 50, 2)]
              )
              self._terminated = False
              self.count = 0
  
          def step(self, action):
              new_state = self._state.copy()
              new_state[:2] += self.moves[action]
              self._state = new_state
              self.count += 1
              reward = 0
              if np.array_equal(new_state[:2], new_state[2:]):
                  self._terminated = True
                  reward = 1
              elif self.count == 1000:
                  self._terminated = True
              return reward
  
          def apply_hindsight(self, states, actions, goal):
              new_states = []
              rewards = []
              goal_pos = goal[:2]
              for state, action in zip(states, actions):
                  new_state = state.copy()
                  new_state[2:] = goal_pos
                  new_states.append(new_state)
                  pos_after_action = new_state[:2] + self.moves[action]
                  if np.array_equal(pos_after_action, goal_pos):
                      rewards.append(1)
                      break
                  else:
                      rewards.append(0)
              return new_states, rewards
  
      # A simple policy with two hidden layers.
  
      class TestPolicy(dc.rl.Policy):
          def __init__(self):
              super(TestPolicy, self).__init__(["action_prob", "value"])
  
          def create_model(self, **kwargs):
              state = Input(shape=(4,))
              dense1 = Dense(8, activation=tf.nn.relu)(state)
              dense2 = Dense(8, activation=tf.nn.relu)(dense1)
              output = Dense(
                  4, activation=tf.nn.softmax, use_bias=False
              )(dense2)
              value = Dense(1)(dense2)
              return tf.keras.Model(
                  inputs=state, outputs=[output, value]
              )
  
      # Optimize it.
  
      env = TestEnvironment()
      ppo = dc.rl.PPO(
          env,
          TestPolicy(),
          use_hindsight=True,
          optimization_epochs=1,
          batch_size=0,
          optimizer=Adam(learning_rate=0.001),
      )
      ppo.fit(1500000)
  
      # Try running it a few times and see if it succeeds.
  
      pass_count = 0
      for i in range(5):
          env.reset()
          while not env.terminated:
              env.step(ppo.select_action(env.state))
          if np.array_equal(env.state[:2], env.state[2:]):
              pass_count += 1
      assert pass_count >= 3
  
file_path: deepchem/deepchem/rl/tests/test_ppo.py
imports: import unittest
  import pytest
  import numpy
  from flaky import flaky
  import deepchem
  from deepchem.models.optimizers import Adam
label_desc: Use image node_id 3 for calling the TestPPO obj's underlying member method code with example usage: obj.test_hindsight() and returns: reward, new_states,  rewards, tf