class_docstr: Compilation and learning tests for the Fitted-Q Evaluation model
class_docstr_tok: ["Compilation","and","learning","tests","for","the","Fitted-Q","Evaluation","model"]
docstr: Test that FQE converges to the true Q-values for an optimal trajectory
  
  self.batch is deterministic since it is collected under a CliffWalkingWallPolicy
  with epsilon = 0.0; check that FQE converges to the true Q-values for self.batch
docstr_tok: ["Test","that","FQE","converges","to","the","true","Q-values","for","an","optimal","trajectory","self.batch","is","deterministic","since","it","is","collected","under","a","CliffWalkingWallPolicy","with","epsilon","=","0.0",";","check","that","FQE","converges","to","the","true","Q-values","for","self.batch"]
code: def test_fqe_optimal_convergence(self):
      """Test that FQE converges to the true Q-values for an optimal trajectory
  
      self.batch is deterministic since it is collected under a CliffWalkingWallPolicy
      with epsilon = 0.0; check that FQE converges to the true Q-values for self.batch
      """
  
      # If self.batch["rewards"] =
      #   [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 10],
      # and gamma = 0.99, the discounted returns i.e. optimal Q-values are as follows:
  
      q_values = np.zeros(len(self.batch["rewards"]), dtype=float)
      q_values[-1] = self.batch["rewards"][-1]
      for t in range(len(self.batch["rewards"]) - 2, -1, -1):
          q_values[t] = (
              self.batch["rewards"][t] + self.gamma * q_values[t + 1]
          )
  
      print(q_values)
  
      q_model_config = {
          "polyak_coef": 1.0,
          "model_config": {
              "fcnet_hiddens": [],
              "activation": "linear",
          },
          "lr": 0.01,
          "n_iters": 5000,
      }
  
      fqe = FQETorchModel(
          policy=self.policy,
          gamma=self.gamma,
          **q_model_config,
      )
      losses = fqe.train(self.batch)
      print(losses[-10:])
      estimates = fqe.estimate_v(self.batch)
      print(estimates)
      check(estimates, q_values, decimals=1)
  
imports: from typing import TYPE_CHECKING, Tuple
  import copy
  import gymnasium
  import numpy
  import os
  import shutil
  import tempfile
  import pandas
  from pathlib import Path
  import unittest
  import ray
  from ray.data import read_json
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.dqn import DQNConfig
  from ray.rllib.examples.env.cliff_walking_wall_env import CliffWalkingWallEnv
  from ray.rllib.examples.policy.cliff_walking_wall_policy import CliffWalkingWallPolicy
  from ray.rllib.offline.dataset_reader import DatasetReader
  from ray.rllib.offline.estimators import DirectMethod, DoublyRobust, ImportanceSampling, WeightedImportanceSampling
  from ray.rllib.offline.estimators.fqe_torch_model import FQETorchModel
  from ray.rllib.policy.sample_batch import SampleBatch, concat_samples
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy
  from ray.rllib.utils.test_utils import check
label_desc: Use image node_id 4 for calling the TestFQE obj's underlying member method code with example usage: obj.test_fqe_optimal_convergence() without return types