class_docstr: Encapsulates multiple torch state action critics for training in structured environments.
  
  :param networks: Mapping of value functions (critic) to encapsulate.
  :param num_policies: The number of corresponding policies.
  :param device: Device the policy should be located on (cpu or cuda)
  :param only_discrete_spaces: A dict specifying if the action spaces w.r.t. the step only hold discrete action
                               spaces.
class_docstr_tok: ["Encapsulates","multiple","torch","state","action","critics","for","training","in","structured","environments",".",":","param","networks",":","Mapping","of","value","functions","(","critic",")","to","encapsulate",".",":","param","num_policies",":","The","number","of","corresponding","policies",".",":","param","device",":","Device","the","policy","should","be","located","on","(","cpu","or","cuda",")",":","param","only_discrete_spaces",":","A","dict","specifying","if","the","action","spaces","w.r.t",".","the","step","only","hold","discrete","action","spaces","."]
code: def __init__(
      self,
      networks: Mapping[Union[str, int], nn.Module],
      num_policies: int,
      device: str,
      only_discrete_spaces: Dict[Union[str, int], bool],
      action_spaces_dict: Dict[Union[str, int], spaces.Dict],
  ):
      # TODO: make this a hyperparameter
      self._num_critics_per_step = 2
      self.step_critic_keys = list(networks.keys())
  
      # Add target networks
      networks = dict(networks)
      networks.update(
          {
              (old_name, self.target_key): copy.deepcopy(critic)
              for old_name, critic in networks.items()
          }
      )
  
      # Add multi q networks
      self.critic_key_mapping = dict()
      new_networks = dict()
      for old_name, step_critic in networks.items():
          tmp_critics = {
              (old_name, idx): copy.deepcopy(step_critic)
              for idx in range(self._num_critics_per_step)
          }
          self.critic_key_mapping[old_name] = list(tmp_critics.keys())
          new_networks.update(tmp_critics)
      self.networks = new_networks
  
      self.num_policies = num_policies
      self.only_discrete_spaces = only_discrete_spaces
      self._preprocessors = dict()
      for step_key, only_discrete in self.only_discrete_spaces.items():
          if not only_discrete:
              discrete_spaces_keys = [
                  key
                  for key, value in action_spaces_dict[
                      step_key
                  ].spaces.items()
                  if isinstance(value, spaces.Discrete)
              ]
              for action_key in discrete_spaces_keys:
                  for critic_key in [
                      step_key,
                      (step_key, self.target_key),
                  ]:
                      if critic_key not in self._preprocessors:
                          self._preprocessors[critic_key] = dict()
                      self._preprocessors[critic_key][
                          action_key
                      ] = OneHotPreProcessor(
                          action_spaces_dict[step_key][action_key]
                      )
  
      TorchModel.__init__(self, device=device)
      self.re_init_networks()
  
imports: import copy
  import itertools
  from abc import abstractmethod
  from typing import Mapping, Union, List, Dict
  import torch
  from gym import spaces
  from torch import nn
  from torch.distributions.utils import logits_to_probs
  from maze.core.agent.state_action_critic import StateActionCritic
  from maze.core.agent.torch_model import TorchModel
  from maze.core.annotations import override
  from maze.core.wrappers.observation_preprocessing.preprocessors.one_hot import OneHotPreProcessor
  from maze.perception.blocks.inference import InferenceBlock
  from maze.perception.perception_utils import flatten_spaces, convert_to_numpy, convert_to_torch
  from maze.perception.weight_init import make_module_init_normc
  from maze.utils.bcolors import BColors
label_desc: Use image node_id 1 to create a new TorchStateActionCritic object from inherited base classes: TorchModel, StateActionCritic with example: obj = TorchStateActionCritic(networks, num_policies, device, only_discrete_spaces, action_spaces_dict)