class_docstr: The step-wise WIS estimator.
  
  Let s_t, a_t, and r_t be the state, action, and reward at timestep t.
  
  For behavior policy \pi_b and evaluation policy \pi_e, define the
  cumulative importance ratio at timestep t as:
  p_t = \sum_{t'=0}^t (\pi_e(a_{t'} | s_{t'}) / \pi_b(a_{t'} | s_{t'})).
  
  Define the average importance ratio over episodes i in the dataset D as:
  w_t = \sum_{i \in D} p^(i)_t / |D|
  
  This estimator computes the expected return for \pi_e for an episode as:
  V^{\pi_e}(s_0) = \E[\sum_t \gamma ^ {t} * (p_t / w_t) * r_t]
  and returns the mean and standard deviation over episodes.
  
  For more information refer to https://arxiv.org/pdf/1911.06854.pdf
class_docstr_tok: ["The","step-wise","WIS","estimator",".","Let","s_t",",","a_t",",","and","r_t","be","the","state",",","action",",","and","reward","at","timestep","t.","For","behavior","policy","\\pi_b","and","evaluation","policy","\\pi_e",",","define","the","cumulative","importance","ratio","at","timestep","t","as",":","p_t","=","\\sum_","{","t'=0","}","^t","(","\\pi_e","(","a_","{","t","'","}","|","s_","{","t","'","}",")","\/","\\pi_b","(","a_","{","t","'","}","|","s_","{","t","'","}",")",")",".","Define","the","average","importance","ratio","over","episodes","i","in","the","dataset","D","as",":","w_t","=","\\sum_","{","i","\\in","D","}","p^","(","i",")","_t","\/","|D|","This","estimator","computes","the","expected","return","for","\\pi_e","for","an","episode","as",":","V^","{","\\pi_e","}","(","s_0",")","=","\\E","[","\\sum_t","\\gamma","^","{","t","}","*","(","p_t","\/","w_t",")","*","r_t","]","and","returns","the","mean","and","standard","deviation","over","episodes",".","For","more","information","refer","to","https",":","\/\/arxiv.org\/pdf\/1911.06854.pdf"]
code: def peek_on_single_episode(self, episode: SampleBatch) -> None:
      old_prob = episode["action_prob"]
      new_prob = self.compute_action_probs(episode)
  
      # calculate importance ratios
      episode_p = []
      for t in range(episode.count):
          if t == 0:
              pt_prev = 1.0
          else:
              pt_prev = episode_p[t - 1]
          episode_p.append(pt_prev * new_prob[t] / old_prob[t])
  
      for t, p_t in enumerate(episode_p):
          if t >= len(self.cummulative_ips_values):
              self.cummulative_ips_values.append(p_t)
              self.episode_timestep_count.append(1.0)
          else:
              self.cummulative_ips_values[t] += p_t
              self.episode_timestep_count[t] += 1.0
  
      eps_id = episode[SampleBatch.EPS_ID][0]
      if eps_id in self.p:
          raise ValueError(
              f"eps_id {eps_id} was already passed to the peek function. "
              f"Make sure dataset contains only unique episodes with unique ids."
          )
      self.p[eps_id] = episode_p
  
file_path: ray/rllib/offline/estimators/weighted_importance_sampling.py
imports: from typing import Dict, Any, List
  import numpy
  import math
  from ray.data import Dataset
  from ray.rllib.offline.offline_evaluator import OfflineEvaluator
  from ray.rllib.offline.estimators.off_policy_estimator import OffPolicyEstimator
  from ray.rllib.offline.offline_evaluation_utils import remove_time_dim, compute_is_weights
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.policy import Policy
  from ray.rllib.utils.annotations import override, DeveloperAPI
label_desc: Use image node_id 5 for calling the WeightedImportanceSampling obj's underlying member method code with example usage: obj.peek_on_single_episode(episode) without return types