code: def compute_gradient(self, model):
      # model = {'W1':W1, 'W2':W2}
      # given a model, run for one episode and return the parameter
      # to be updated and sum(reward)
      # Reset the game.
      observation = self.env.reset()
      # Note that prev_x is used in computing the difference frame.
      prev_x = None
      xs, hs, dlogps, drs = [], [], [], []
      reward_sum = 0
      done = False
      while not done:
          cur_x = preprocess(observation)
          x = cur_x - prev_x if prev_x is not None else np.zeros(D)
          prev_x = cur_x
  
          # feed difference frames into the network
          # so that it can detect motion
          aprob, h = policy_forward(x, model)
          # Sample an action.
          action = 2 if np.random.uniform() < aprob else 3
  
          # The observation.
          xs.append(x)
          # The hidden state.
          hs.append(h)
          y = 1 if action == 2 else 0  # A "fake label".
          # The gradient that encourages the action that was taken to be
          # taken (see http://cs231n.github.io/neural-networks-2/#losses if
          # confused).
          dlogps.append(y - aprob)
  
          observation, reward, done, info = self.env.step(action)
          reward_sum += reward
  
          # Record reward (has to be done after we call step() to get reward
          # for previous action).
          drs.append(reward)
  
      epx = np.vstack(xs)
      eph = np.vstack(hs)
      epdlogp = np.vstack(dlogps)
      epr = np.vstack(drs)
      # Reset the array memory.
      xs, hs, dlogps, drs = [], [], [], []
  
      # Compute the discounted reward backward through time.
      discounted_epr = discount_rewards(epr)
      # Standardize the rewards to be unit normal (helps control the gradient
      # estimator variance).
      discounted_epr -= np.mean(discounted_epr)
      discounted_epr /= np.std(discounted_epr)
      # Modulate the gradient with advantage (the policy gradient magic
      # happens right here).
      epdlogp *= discounted_epr
      return policy_backward(eph, epx, epdlogp, model), reward_sum
  
file_path: analytics-zoo/pyzoo/zoo/examples/ray_on_spark/rl_pong/rl_pong.py
imports: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  import argparse
  import os
  import time
  import gym
  import numpy
  import ray
  from zoo.orca import init_orca_context, stop_orca_context
  from zoo.orca import OrcaContext
label_desc: Use image node_id 2 for calling the PongEnv obj's underlying member method code with example usage: obj.compute_gradient(model) and returns: policy_backward,  reward_sum