code: def estimate(
      self,
      edp: EvaluationDataPage,
      num_j_steps,
      whether_self_normalize_importance_weights,
  ) -> CpeEstimate:
      # For details, visit https://arxiv.org/pdf/1604.00923.pdf Section 5, 7, 8
      assert edp.model_values is not None
      (
          actions,
          rewards,
          logged_propensities,
          target_propensities,
          estimated_q_values,
      ) = WeightedSequentialDoublyRobustEstimator.transform_to_equal_length_trajectories(
          edp.mdp_id,
          edp.action_mask.cpu().numpy(),
          edp.logged_rewards.cpu().numpy().flatten(),
          edp.logged_propensities.cpu().numpy().flatten(),
          edp.model_propensities.cpu().numpy(),
          # pyre-ignore [16]: Optional type has no attribute `cpu`
          edp.model_values.cpu().numpy(),
      )
  
      num_trajectories = actions.shape[0]
      trajectory_length = actions.shape[1]
  
      j_steps = [float("inf")]
  
      if num_j_steps > 1:
          j_steps.append(-1)
      if num_j_steps > 2:
          interval = trajectory_length // (num_j_steps - 1)
          j_steps.extend(
              [i * interval for i in range(1, num_j_steps - 1)]
          )
  
      target_propensity_for_logged_action = np.sum(
          np.multiply(target_propensities, actions), axis=2
      )
      estimated_q_values_for_logged_action = np.sum(
          np.multiply(estimated_q_values, actions), axis=2
      )
      estimated_state_values = np.sum(
          np.multiply(target_propensities, estimated_q_values), axis=2
      )
  
      importance_weights = (
          target_propensity_for_logged_action / logged_propensities
      )
      importance_weights = np.cumprod(importance_weights, axis=1)
      importance_weights = WeightedSequentialDoublyRobustEstimator.normalize_importance_weights(
          importance_weights, whether_self_normalize_importance_weights
      )
  
      importance_weights_one_earlier = (
          np.ones([num_trajectories, 1]) * 1.0 / num_trajectories
      )
      importance_weights_one_earlier = np.hstack(
          [importance_weights_one_earlier, importance_weights[:, :-1]]
      )
  
      discounts = np.logspace(
          start=0,
          stop=trajectory_length - 1,
          num=trajectory_length,
          base=self.gamma,
      )
  
      j_step_return_trajectories = []
      for j_step in j_steps:
          j_step_return_trajectories.append(
              WeightedSequentialDoublyRobustEstimator.calculate_step_return(
                  rewards,
                  discounts,
                  importance_weights,
                  importance_weights_one_earlier,
                  estimated_state_values,
                  estimated_q_values_for_logged_action,
                  j_step,
              )
          )
      j_step_return_trajectories = np.array(j_step_return_trajectories)
  
      j_step_returns = np.sum(j_step_return_trajectories, axis=1)
  
      if len(j_step_returns) == 1:
          weighted_doubly_robust = j_step_returns[0]
          weighted_doubly_robust_std_error = 0.0
      else:
          # break trajectories into several subsets to estimate confidence bounds
          infinite_step_returns = []
          num_subsets = int(
              min(
                  num_trajectories / 2,
                  WeightedSequentialDoublyRobustEstimator.NUM_SUBSETS_FOR_CB_ESTIMATES,
              )
          )
          interval = num_trajectories / num_subsets
          for i in range(num_subsets):
              trajectory_subset = np.arange(
                  int(i * interval), int((i + 1) * interval)
              )
              importance_weights = (
                  target_propensity_for_logged_action[trajectory_subset]
                  / logged_propensities[trajectory_subset]
              )
              importance_weights = np.cumprod(
                  importance_weights, axis=1
              )
              importance_weights = WeightedSequentialDoublyRobustEstimator.normalize_importance_weights(
                  importance_weights,
                  whether_self_normalize_importance_weights,
              )
              importance_weights_one_earlier = (
                  np.ones([len(trajectory_subset), 1])
                  * 1.0
                  / len(trajectory_subset)
              )
              importance_weights_one_earlier = np.hstack(
                  [
                      importance_weights_one_earlier,
                      importance_weights[:, :-1],
                  ]
              )
              infinite_step_return = np.sum(
                  WeightedSequentialDoublyRobustEstimator.calculate_step_return(
                      rewards[trajectory_subset],
                      discounts,
                      importance_weights,
                      importance_weights_one_earlier,
                      estimated_state_values[trajectory_subset],
                      estimated_q_values_for_logged_action[
                          trajectory_subset
                      ],
                      float("inf"),
                  )
              )
              infinite_step_returns.append(infinite_step_return)
  
          # Compute weighted_doubly_robust mean point estimate using all data
          weighted_doubly_robust = (
              self.compute_weighted_doubly_robust_point_estimate(
                  j_steps,
                  num_j_steps,
                  j_step_returns,
                  infinite_step_returns,
                  j_step_return_trajectories,
              )
          )
  
          # Use bootstrapping to compute weighted_doubly_robust standard error
          bootstrapped_means = []
          sample_size = int(
              WeightedSequentialDoublyRobustEstimator.BOOTSTRAP_SAMPLE_PCT
              * num_subsets
          )
          for _ in range(
              WeightedSequentialDoublyRobustEstimator.NUM_BOOTSTRAP_SAMPLES
          ):
              random_idxs = np.random.choice(
                  num_j_steps, sample_size, replace=False
              )
              random_idxs.sort()
              wdr_estimate = self.compute_weighted_doubly_robust_point_estimate(
                  j_steps=[j_steps[i] for i in random_idxs],
                  num_j_steps=sample_size,
                  j_step_returns=j_step_returns[random_idxs],
                  infinite_step_returns=infinite_step_returns,
                  j_step_return_trajectories=j_step_return_trajectories[
                      random_idxs
                  ],
              )
              bootstrapped_means.append(wdr_estimate)
          weighted_doubly_robust_std_error = np.std(bootstrapped_means)
  
      episode_values = np.sum(np.multiply(rewards, discounts), axis=1)
      logged_policy_score = np.nanmean(episode_values)
      if logged_policy_score < 1e-6:
          logger.warning(
              "Can't normalize WSDR-CPE because of small or negative logged_policy_score"
          )
          return CpeEstimate(
              raw=weighted_doubly_robust,
              normalized=0.0,
              raw_std_error=weighted_doubly_robust_std_error,
              normalized_std_error=0.0,
          )
  
      return CpeEstimate(
          raw=weighted_doubly_robust,
          normalized=weighted_doubly_robust / logged_policy_score,
          raw_std_error=weighted_doubly_robust_std_error,
          normalized_std_error=weighted_doubly_robust_std_error
          / logged_policy_score,
      )
  
file_path: ReAgent/reagent/evaluation/weighted_sequential_doubly_robust_estimator.py
imports: import itertools
  import logging
  import numpy
  import scipy
  from reagent.evaluation.cpe import CpeEstimate
  from reagent.evaluation.evaluation_data_page import EvaluationDataPage
label_desc: Use image node_id 2 for calling the WeightedSequentialDoublyRobustEstimator obj's underlying member method code with example usage: obj.estimate(edp, num_j_steps, whether__normalize_importance_weights) and returns: CpeEstimate, CpeEstimate