class_docstr: Compilation and learning tests for the Fitted-Q Evaluation model
class_docstr_tok: ["Compilation","and","learning","tests","for","the","Fitted-Q","Evaluation","model"]
code: def setUpClass(cls) -> None:
      ray.init()
  
      env = CliffWalkingWallEnv()
      cls.policy = CliffWalkingWallPolicy(
          observation_space=env.observation_space,
          action_space=env.action_space,
          config={},
      )
      cls.gamma = 0.99
  
      # Collect single episode under optimal policy
      obs_batch = []
      new_obs = []
      actions = []
      action_prob = []
      rewards = []
      terminateds = []
      truncateds = []
  
      obs, info = env.reset()
  
      terminated = truncated = False
      while not terminated and not truncated:
          obs_batch.append(obs)
          act, _, extra = cls.policy.compute_single_action(obs)
          actions.append(act)
          action_prob.append(extra["action_prob"])
          obs, rew, terminated, truncated, _ = env.step(act)
          new_obs.append(obs)
          rewards.append(rew)
          terminateds.append(terminated)
          truncateds.append(truncated)
  
      cls.batch = SampleBatch(
          obs=obs_batch,
          actions=actions,
          action_prob=action_prob,
          rewards=rewards,
          terminateds=terminateds,
          truncateds=truncateds,
          new_obs=new_obs,
      )
  
imports: from typing import TYPE_CHECKING, Tuple
  import copy
  import gymnasium
  import numpy
  import os
  import shutil
  import tempfile
  import pandas
  from pathlib import Path
  import unittest
  import ray
  from ray.data import read_json
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.dqn import DQNConfig
  from ray.rllib.examples.env.cliff_walking_wall_env import CliffWalkingWallEnv
  from ray.rllib.examples.policy.cliff_walking_wall_policy import CliffWalkingWallPolicy
  from ray.rllib.offline.dataset_reader import DatasetReader
  from ray.rllib.offline.estimators import DirectMethod, DoublyRobust, ImportanceSampling, WeightedImportanceSampling
  from ray.rllib.offline.estimators.fqe_torch_model import FQETorchModel
  from ray.rllib.policy.sample_batch import SampleBatch, concat_samples
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy
  from ray.rllib.utils.test_utils import check
label_desc: Use image node_id 1 for calling the TestFQE obj's underlying member method code with example usage: obj.setUpClass(cls) without return types