class_docstr: PyTorch policy class used with APPO.
code: def __init__(self, observation_space, action_space, config):
      config = dict(
          ray.rllib.algorithms.appo.appo.APPOConfig().to_dict(),
          **config,
      )
  
      # If Learner API is used, we don't need any loss-specific mixins.
      # However, we also would like to avoid creating special Policy-subclasses
      # for this as the entire Policy concept will soon not be used anymore with
      # the new Learner- and RLModule APIs.
      if not config.get("_enable_new_api_stack", False):
          # Although this is a no-op, we call __init__ here to make it clear
          # that base.__init__ will use the make_model() call.
          VTraceOptimizer.__init__(self)
  
      LearningRateSchedule.__init__(
          self, config["lr"], config["lr_schedule"]
      )
  
      TorchPolicyV2.__init__(
          self,
          observation_space,
          action_space,
          config,
          max_seq_len=config["model"]["max_seq_len"],
      )
  
      EntropyCoeffSchedule.__init__(
          self,
          config["entropy_coeff"],
          config["entropy_coeff_schedule"],
      )
      ValueNetworkMixin.__init__(self, config)
      KLCoeffMixin.__init__(self, config)
  
      self._initialize_loss_from_dummy_batch()
  
      # Initiate TargetNetwork ops after loss initialization.
      TargetNetworkMixin.__init__(self)
  
imports: import gymnasium
  import numpy
  import logging
  from typing import Any, Dict, List, Optional, Type, Union
  import ray
  from ray.rllib.algorithms.appo.utils import make_appo_models
  import ray.rllib.algorithms.impala.vtrace_torch
  from ray.rllib.algorithms.impala.impala_torch_policy import make_time_major, VTraceOptimizer
  from ray.rllib.evaluation.episode import Episode
  from ray.rllib.evaluation.postprocessing import compute_bootstrap_value, compute_gae_for_sample_batch, Postprocessing
  from ray.rllib.models.action_dist import ActionDistribution
  from ray.rllib.models.modelv2 import ModelV2
  from ray.rllib.models.torch.torch_action_dist import TorchDistributionWrapper, TorchCategorical
  from ray.rllib.models.torch.torch_modelv2 import TorchModelV2
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.policy.torch_mixins import EntropyCoeffSchedule, LearningRateSchedule, KLCoeffMixin, ValueNetworkMixin, TargetNetworkMixin
  from ray.rllib.policy.torch_policy_v2 import TorchPolicyV2
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.framework import try_import_torch
  from ray.rllib.utils.numpy import convert_to_numpy
  from ray.rllib.utils.torch_utils import apply_grad_clipping, explained_variance, global_norm, sequence_mask
  from ray.rllib.utils.typing import TensorType
label_desc: Use image node_id 1 to create a new APPOTorchPolicy object from inherited base classes: VTraceOptimizer, LearningRateSchedule, EntropyCoeffSchedule, KLCoeffMixin, ValueNetworkMixin, TargetNetworkMixin, TorchPolicyV2 with example: obj = APPOTorchPolicy(observation_space, action_space, config)