class_docstr: Compilation tests for using OPE both standalone and in an RLlib Algorithm
class_docstr_tok: ["Compilation","tests","for","using","OPE","both","standalone","and","in","an","RLlib","Algorithm"]
code: def setUpClass(cls):
      ray.init()
      seed = 42
      np.random.seed(seed)
  
      rllib_dir = Path(__file__).parent.parent.parent.parent
      train_data = os.path.join(
          rllib_dir, "tests/data/cartpole/small.json"
      )
  
      env_name = "CartPole-v1"
      cls.gamma = 0.99
      n_episodes = 3
      cls.q_model_config = {"n_iters": 160}
  
      cls.config_dqn_on_cartpole = (
          DQNConfig()
          .environment(env=env_name)
          .framework("torch")
          .rollouts(batch_mode="complete_episodes")
          .offline_data(
              input_="dataset",
              input_config={"format": "json", "paths": train_data},
          )
          .evaluation(
              evaluation_interval=1,
              evaluation_duration=n_episodes,
              evaluation_num_workers=1,
              evaluation_duration_unit="episodes",
              off_policy_estimation_methods={
                  "is": {
                      "type": ImportanceSampling,
                      "epsilon_greedy": 0.1,
                  },
                  "wis": {
                      "type": WeightedImportanceSampling,
                      "epsilon_greedy": 0.1,
                  },
                  "dm_fqe": {
                      "type": DirectMethod,
                      "epsilon_greedy": 0.1,
                  },
                  "dr_fqe": {
                      "type": DoublyRobust,
                      "epsilon_greedy": 0.1,
                  },
              },
          )
          .resources(num_gpus=int(os.environ.get("RLLIB_NUM_GPUS", 0)))
      )
  
      num_rollout_workers = 4
      dsize = num_rollout_workers * 1024
      feature_dim = 64
      action_dim = 8
  
      data = {
          SampleBatch.OBS: np.random.randn(dsize, 1, feature_dim),
          SampleBatch.ACTIONS: np.random.randint(
              0, action_dim, dsize
          ).reshape(-1, 1),
          SampleBatch.REWARDS: np.random.rand(dsize).reshape(-1, 1),
          SampleBatch.ACTION_PROB: 1 / action_dim * np.ones((dsize, 1)),
      }
      cls.train_df = pd.DataFrame({k: list(v) for k, v in data.items()})
      cls.train_df["type"] = "SampleBatch"
  
      train_ds = ray.data.from_pandas(cls.train_df).repartition(
          num_rollout_workers
      )
  
      cls.dqn_on_fake_ds = (
          DQNConfig()
          .environment(
              observation_space=gym.spaces.Box(-1, 1, (feature_dim,)),
              action_space=gym.spaces.Discrete(action_dim),
          )
          .rollouts(num_rollout_workers=num_rollout_workers)
          .framework("torch")
          # .rollouts(num_rollout_workers=num_rollout_workers)
          .offline_data(
              input_="dataset",
              input_config={"loader_fn": lambda: train_ds},
          )
          .evaluation(
              evaluation_num_workers=num_rollout_workers,
              ope_split_batch_by_episode=False,
          )
          # make the policy deterministic
          .training(categorical_distribution_temperature=1e-20)
          .debugging(seed=seed)
      )
  
      # Read n episodes of data, assuming that one line is one episode.
      reader = DatasetReader(read_json(train_data))
      batches = [reader.next() for _ in range(n_episodes)]
      cls.batch = concat_samples(batches)
      cls.n_episodes = len(cls.batch.split_by_episode())
      print("Episodes:", cls.n_episodes, "Steps:", cls.batch.count)
  
imports: from typing import TYPE_CHECKING, Tuple
  import copy
  import gymnasium
  import numpy
  import os
  import shutil
  import tempfile
  import pandas
  from pathlib import Path
  import unittest
  import ray
  from ray.data import read_json
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.dqn import DQNConfig
  from ray.rllib.examples.env.cliff_walking_wall_env import CliffWalkingWallEnv
  from ray.rllib.examples.policy.cliff_walking_wall_policy import CliffWalkingWallPolicy
  from ray.rllib.offline.dataset_reader import DatasetReader
  from ray.rllib.offline.estimators import DirectMethod, DoublyRobust, ImportanceSampling, WeightedImportanceSampling
  from ray.rllib.offline.estimators.fqe_torch_model import FQETorchModel
  from ray.rllib.policy.sample_batch import SampleBatch, concat_samples
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy
  from ray.rllib.utils.test_utils import check
label_desc: Use image node_id 1 for calling the TestOPE obj's underlying member method code with example usage: obj.setUpClass(cls) without return types