class_docstr: Pytorch implementation of the Fitted Q-Evaluation (FQE) model from
  https://arxiv.org/abs/1911.06854
class_docstr_tok: ["Pytorch","implementation","of","the","Fitted","Q-Evaluation","(","FQE",")","model","from","https",":","\/\/arxiv.org\/abs\/1911.06854"]
docstr: Args:
      policy: Policy to evaluate.
      gamma: Discount factor of the environment.
      model_config: The ModelConfigDict for self.q_model, defaults to:
          {
              "fcnet_hiddens": [8, 8],
              "fcnet_activation": "relu",
              "vf_share_layers": True,
          },
      n_iters: Number of gradient steps to run on batch, defaults to 1
      lr: Learning rate for Adam optimizer
      min_loss_threshold: Early stopping if mean loss < min_loss_threshold
      clip_grad_norm: Clip loss gradients to this maximum value
      minibatch_size: Minibatch size for training Q-function;
          if None, train on the whole batch
      polyak_coef: Polyak averaging factor for target Q-function
docstr_tok: ["Args",":","policy",":","Policy","to","evaluate",".","gamma",":","Discount","factor","of","the","environment",".","model_config",":","The","ModelConfigDict","for","self.q_model",",","defaults","to",":","{","``","fcnet_hiddens","''",":","[","8",",","8","]",",","``","fcnet_activation","''",":","``","relu","''",",","``","vf_share_layers","''",":","True",",","}",",","n_iters",":","Number","of","gradient","steps","to","run","on","batch",",","defaults","to","1","lr",":","Learning","rate","for","Adam","optimizer","min_loss_threshold",":","Early","stopping","if","mean","loss","<","min_loss_threshold","clip_grad_norm",":","Clip","loss","gradients","to","this","maximum","value","minibatch_size",":","Minibatch","size","for","training","Q-function",";","if","None",",","train","on","the","whole","batch","polyak_coef",":","Polyak","averaging","factor","for","target","Q-function"]
code: def __init__(
      self,
      policy: Policy,
      gamma: float,
      model_config: ModelConfigDict = None,
      n_iters: int = 1,
      lr: float = 1e-3,
      min_loss_threshold: float = 1e-4,
      clip_grad_norm: float = 100.0,
      minibatch_size: int = None,
      polyak_coef: float = 1.0,
  ) -> None:
      """
      Args:
          policy: Policy to evaluate.
          gamma: Discount factor of the environment.
          model_config: The ModelConfigDict for self.q_model, defaults to:
              {
                  "fcnet_hiddens": [8, 8],
                  "fcnet_activation": "relu",
                  "vf_share_layers": True,
              },
          n_iters: Number of gradient steps to run on batch, defaults to 1
          lr: Learning rate for Adam optimizer
          min_loss_threshold: Early stopping if mean loss < min_loss_threshold
          clip_grad_norm: Clip loss gradients to this maximum value
          minibatch_size: Minibatch size for training Q-function;
              if None, train on the whole batch
          polyak_coef: Polyak averaging factor for target Q-function
      """
      self.policy = policy
      assert isinstance(
          policy.action_space, Discrete
      ), f"{self.__class__.__name__} only supports discrete action spaces!"
      self.gamma = gamma
      self.observation_space = policy.observation_space
      self.action_space = policy.action_space
  
      if model_config is None:
          model_config = {
              "fcnet_hiddens": [32, 32, 32],
              "fcnet_activation": "relu",
              "vf_share_layers": True,
          }
      self.model_config = model_config
  
      self.device = self.policy.device
      self.q_model: TorchModelV2 = ModelCatalog.get_model_v2(
          self.observation_space,
          self.action_space,
          self.action_space.n,
          model_config,
          framework="torch",
          name="TorchQModel",
      ).to(self.device)
  
      self.target_q_model: TorchModelV2 = ModelCatalog.get_model_v2(
          self.observation_space,
          self.action_space,
          self.action_space.n,
          model_config,
          framework="torch",
          name="TargetTorchQModel",
      ).to(self.device)
  
      self.n_iters = n_iters
      self.lr = lr
      self.min_loss_threshold = min_loss_threshold
      self.clip_grad_norm = clip_grad_norm
      self.minibatch_size = minibatch_size
      self.polyak_coef = polyak_coef
      self.optimizer = torch.optim.Adam(
          self.q_model.variables(), self.lr
      )
      initializer = get_initializer("xavier_uniform", framework="torch")
      # Hard update target
      self.update_target(polyak_coef=1.0)
  
      def f(m):
          if isinstance(m, nn.Linear):
              initializer(m.weight)
  
      self.initializer = f
  
imports: from typing import Dict, Any
  from ray.rllib.models.utils import get_initializer
  from ray.rllib.policy import Policy
  from ray.rllib.models.catalog import ModelCatalog
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import DeveloperAPI
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.annotations import is_overridden
  from ray.rllib.utils.typing import ModelConfigDict, TensorType
  from gymnasium.spaces import Discrete
label_desc: Use image node_id 1 to create a new FQETorchModel object  with example: obj = FQETorchModel(policy, gamma, model_config, n_iters, lr, min_loss_threshold, clip_grad_norm, minibatch_size, polyak_coef)