hunk.code_window:             loss_inputs=self.loss_inputs,
               update_ops=q_batchnorm_update_ops)
           self.sess.run(tf.global_variables_initializer())
       @override(TFPolicyGraph)
       def optimizer(self):
           return tf.train.AdamOptimizer(
  
hunk.labels: keep keep keep add keep keep keep
hunk.after_edit:         self.stats_fetches = dict({
               "cur_lr": tf.cast(self.cur_lr, tf.float64),
           }, **self.loss.stats)
  
hunk.file_path: python/ray/rllib/agents/dqn/dqn_policy_graph.py
hunk.type: add
file: from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  
  from ray.rllib.offline.off_policy_estimator import OffPolicyEstimator, \
      OffPolicyEstimate
  from ray.rllib.utils.annotations import override
  
  
  class ImportanceSamplingEstimator(OffPolicyEstimator):
      """The step-wise IS estimator.
  
      Step-wise IS estimator described in https://arxiv.org/pdf/1511.03722.pdf"""
  
      def __init__(self, policy, gamma):
          OffPolicyEstimator.__init__(self, policy, gamma)
  
      @override(OffPolicyEstimator)
      def estimate(self, batch):
          self.check_can_estimate_for(batch)
  
          rewards, old_prob = batch["rewards"], batch["action_prob"]
          new_prob = self.action_prob(batch)
  
          # calculate importance ratios
          p = []
          for t in range(batch.count - 1):
              if t == 0:
                  pt_prev = 1.0
              else:
                  pt_prev = p[t - 1]
              p.append(pt_prev * new_prob[t] / old_prob[t])
  
          # calculate stepwise IS estimate
          V_prev, V_step_IS = 0.0, 0.0
          for t in range(batch.count - 1):
              V_prev += rewards[t] * self.gamma**t
              V_step_IS += p[t] * rewards[t] * self.gamma**t
  
          estimation = OffPolicyEstimate(
              "is", {
                  "V_prev": V_prev,
                  "V_step_IS": V_step_IS,
                  "V_gain_est": V_step_IS / max(1e-8, V_prev),
              })
          return estimation
  