class_docstr: The step-wise WIS estimator.
  
  Let s_t, a_t, and r_t be the state, action, and reward at timestep t.
  
  For behavior policy \pi_b and evaluation policy \pi_e, define the
  cumulative importance ratio at timestep t as:
  p_t = \sum_{t'=0}^t (\pi_e(a_{t'} | s_{t'}) / \pi_b(a_{t'} | s_{t'})).
  
  Define the average importance ratio over episodes i in the dataset D as:
  w_t = \sum_{i \in D} p^(i)_t / |D|
  
  This estimator computes the expected return for \pi_e for an episode as:
  V^{\pi_e}(s_0) = \E[\sum_t \gamma ^ {t} * (p_t / w_t) * r_t]
  and returns the mean and standard deviation over episodes.
  
  For more information refer to https://arxiv.org/pdf/1911.06854.pdf
class_docstr_tok: ["The","step-wise","WIS","estimator",".","Let","s_t",",","a_t",",","and","r_t","be","the","state",",","action",",","and","reward","at","timestep","t.","For","behavior","policy","\\pi_b","and","evaluation","policy","\\pi_e",",","define","the","cumulative","importance","ratio","at","timestep","t","as",":","p_t","=","\\sum_","{","t'=0","}","^t","(","\\pi_e","(","a_","{","t","'","}","|","s_","{","t","'","}",")","\/","\\pi_b","(","a_","{","t","'","}","|","s_","{","t","'","}",")",")",".","Define","the","average","importance","ratio","over","episodes","i","in","the","dataset","D","as",":","w_t","=","\\sum_","{","i","\\in","D","}","p^","(","i",")","_t","\/","|D|","This","estimator","computes","the","expected","return","for","\\pi_e","for","an","episode","as",":","V^","{","\\pi_e","}","(","s_0",")","=","\\E","[","\\sum_t","\\gamma","^","{","t","}","*","(","p_t","\/","w_t",")","*","r_t","]","and","returns","the","mean","and","standard","deviation","over","episodes",".","For","more","information","refer","to","https",":","\/\/arxiv.org\/pdf\/1911.06854.pdf"]
docstr: Computes the weighted importance sampling estimate on a dataset.
  
  Note: This estimate works for both continuous and discrete action spaces.
  
  Args:
      dataset: Dataset to compute the estimate on. Each record in dataset should
          include the following columns: `obs`, `actions`, `action_prob` and
          `rewards`. The `obs` on each row shoud be a vector of D dimensions.
      n_parallelism: Number of parallel workers to use for the computation.
  
  Returns:
      Dictionary with the following keys:
          v_target: The weighted importance sampling estimate.
          v_behavior: The behavior policy estimate.
          v_gain_mean: The mean of the gain of the target policy over the
              behavior policy.
          v_gain_ste: The standard error of the gain of the target policy over
              the behavior policy.
docstr_tok: ["Computes","the","weighted","importance","sampling","estimate","on","a","dataset",".","Note",":","This","estimate","works","for","both","continuous","and","discrete","action","spaces",".","Args",":","dataset",":","Dataset","to","compute","the","estimate","on",".","Each","record","in","dataset","should","include","the","following","columns",":","`","obs","`",",","`","actions","`",",","`","action_prob","`","and","`","rewards","`",".","The","`","obs","`","on","each","row","shoud","be","a","vector","of","D","dimensions",".","n_parallelism",":","Number","of","parallel","workers","to","use","for","the","computation",".","Returns",":","Dictionary","with","the","following","keys",":","v_target",":","The","weighted","importance","sampling","estimate",".","v_behavior",":","The","behavior","policy","estimate",".","v_gain_mean",":","The","mean","of","the","gain","of","the","target","policy","over","the","behavior","policy",".","v_gain_ste",":","The","standard","error","of","the","gain","of","the","target","policy","over","the","behavior","policy","."]
code: def estimate_on_dataset(
      self, dataset: Dataset, *, n_parallelism: int = ...
  ) -> Dict[str, Any]:
      """Computes the weighted importance sampling estimate on a dataset.
  
      Note: This estimate works for both continuous and discrete action spaces.
  
      Args:
          dataset: Dataset to compute the estimate on. Each record in dataset should
              include the following columns: `obs`, `actions`, `action_prob` and
              `rewards`. The `obs` on each row shoud be a vector of D dimensions.
          n_parallelism: Number of parallel workers to use for the computation.
  
      Returns:
          Dictionary with the following keys:
              v_target: The weighted importance sampling estimate.
              v_behavior: The behavior policy estimate.
              v_gain_mean: The mean of the gain of the target policy over the
                  behavior policy.
              v_gain_ste: The standard error of the gain of the target policy over
                  the behavior policy.
      """
      # compute the weights and weighted rewards
      batch_size = max(dataset.count() // n_parallelism, 1)
      dataset = dataset.map_batches(
          remove_time_dim, batch_size=batch_size, batch_format="pandas"
      )
      updated_ds = dataset.map_batches(
          compute_is_weights,
          batch_size=batch_size,
          batch_format="pandas",
          fn_kwargs={
              "policy_state": self.policy.get_state(),
              "estimator_class": self.__class__,
          },
      )
      v_target = updated_ds.mean("weighted_rewards") / updated_ds.mean(
          "weights"
      )
      v_behavior = updated_ds.mean("rewards")
      v_gain_mean = v_target / v_behavior
      v_gain_ste = (
          updated_ds.std("weighted_rewards")
          / updated_ds.mean("weights")
          / v_behavior
          / math.sqrt(dataset.count())
      )
  
      return {
          "v_target": v_target,
          "v_behavior": v_behavior,
          "v_gain_mean": v_gain_mean,
          "v_gain_ste": v_gain_ste,
      }
  
file_path: ray/rllib/offline/estimators/weighted_importance_sampling.py
imports: from typing import Dict, Any, List
  import numpy
  import math
  from ray.data import Dataset
  from ray.rllib.offline.offline_evaluator import OfflineEvaluator
  from ray.rllib.offline.estimators.off_policy_estimator import OffPolicyEstimator
  from ray.rllib.offline.offline_evaluation_utils import remove_time_dim, compute_is_weights
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.policy import Policy
  from ray.rllib.utils.annotations import override, DeveloperAPI
label_desc: Use image node_id 6 for calling the WeightedImportanceSampling obj's underlying member method code with example usage: obj.estimate_on_dataset(dataset) and returns: dict