class_docstr: Pytorch implementation of the Fitted Q-Evaluation (FQE) model from
  https://arxiv.org/abs/1911.06854
class_docstr_tok: ["Pytorch","implementation","of","the","Fitted","Q-Evaluation","(","FQE",")","model","from","https",":","\/\/arxiv.org\/abs\/1911.06854"]
docstr: Trains self.q_model using FQE loss on given batch.
  
  Args:
      batch: A SampleBatch of episodes to train on
  
  Returns:
      A list of losses for each training iteration
docstr_tok: ["Trains","self.q_model","using","FQE","loss","on","given","batch",".","Args",":","batch",":","A","SampleBatch","of","episodes","to","train","on","Returns",":","A","list","of","losses","for","each","training","iteration"]
code: def train(self, batch: SampleBatch) -> TensorType:
      """Trains self.q_model using FQE loss on given batch.
  
      Args:
          batch: A SampleBatch of episodes to train on
  
      Returns:
          A list of losses for each training iteration
      """
      losses = []
      minibatch_size = self.minibatch_size or batch.count
      # Copy batch for shuffling
      batch = batch.copy(shallow=True)
      for _ in range(self.n_iters):
          minibatch_losses = []
          batch.shuffle()
          for idx in range(0, batch.count, minibatch_size):
              minibatch = batch[idx : idx + minibatch_size]
              obs = torch.tensor(
                  minibatch[SampleBatch.OBS], device=self.device
              )
              actions = torch.tensor(
                  minibatch[SampleBatch.ACTIONS],
                  device=self.device,
                  dtype=int,
              )
              rewards = torch.tensor(
                  minibatch[SampleBatch.REWARDS], device=self.device
              )
              next_obs = torch.tensor(
                  minibatch[SampleBatch.NEXT_OBS], device=self.device
              )
              dones = torch.tensor(
                  minibatch[SampleBatch.TERMINATEDS],
                  device=self.device,
                  dtype=float,
              )
  
              # Compute Q-values for current obs
              q_values, _ = self.q_model({"obs": obs}, [], None)
              q_acts = torch.gather(
                  q_values, -1, actions.unsqueeze(-1)
              ).squeeze(-1)
  
              next_action_probs = self._compute_action_probs(next_obs)
  
              # Compute Q-values for next obs
              with torch.no_grad():
                  next_q_values, _ = self.target_q_model(
                      {"obs": next_obs}, [], None
                  )
  
              # Compute estimated state value next_v = E_{a ~ pi(s)} [Q(next_obs,a)]
              next_v = torch.sum(
                  next_q_values * next_action_probs, axis=-1
              )
              targets = rewards + (1 - dones) * self.gamma * next_v
              loss = (targets - q_acts) ** 2
              loss = torch.mean(loss)
              self.optimizer.zero_grad()
              loss.backward()
              nn.utils.clip_grad.clip_grad_norm_(
                  self.q_model.variables(), self.clip_grad_norm
              )
              self.optimizer.step()
              minibatch_losses.append(loss.item())
          iter_loss = sum(minibatch_losses) / len(minibatch_losses)
          losses.append(iter_loss)
          if iter_loss < self.min_loss_threshold:
              break
          self.update_target()
      return losses
  
imports: from typing import Dict, Any
  from ray.rllib.models.utils import get_initializer
  from ray.rllib.policy import Policy
  from ray.rllib.models.catalog import ModelCatalog
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.annotations import DeveloperAPI
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.annotations import is_overridden
  from ray.rllib.utils.typing import ModelConfigDict, TensorType
  from gymnasium.spaces import Discrete
label_desc: Use image node_id 2 for calling the FQETorchModel obj's underlying member method code with example usage: obj.train(batch) and returns: losses