docstr: Emulates SAC loss functions for tf and torch.
code: def _sac_loss_helper(
      self, train_batch, weights, ks, log_alpha, fw, gamma, sess
  ):
      """Emulates SAC loss functions for tf and torch."""
      # ks:
      # 0=log_alpha
      # 1=target log-alpha (not used)
  
      # 2=action hidden bias
      # 3=action hidden kernel
      # 4=action out bias
      # 5=action out kernel
  
      # 6=Q hidden bias
      # 7=Q hidden kernel
      # 8=Q out bias
      # 9=Q out kernel
  
      # 14=target Q hidden bias
      # 15=target Q hidden kernel
      # 16=target Q out bias
      # 17=target Q out kernel
      alpha = np.exp(log_alpha)
      # cls = TorchSquashedGaussian if fw == "torch" else SquashedGaussian
      cls = TorchDirichlet if fw == "torch" else Dirichlet
      model_out_t = train_batch[SampleBatch.CUR_OBS]
      model_out_tp1 = train_batch[SampleBatch.NEXT_OBS]
      target_model_out_tp1 = train_batch[SampleBatch.NEXT_OBS]
  
      # get_policy_output
      action_dist_t = cls(
          fc(
              relu(
                  fc(
                      model_out_t,
                      weights[ks[1]],
                      weights[ks[0]],
                      framework=fw,
                  )
              ),
              weights[ks[9]],
              weights[ks[8]],
          ),
          None,
      )
      policy_t = action_dist_t.deterministic_sample()
      log_pis_t = action_dist_t.logp(policy_t)
      if sess:
          log_pis_t = sess.run(log_pis_t)
          policy_t = sess.run(policy_t)
      log_pis_t = np.expand_dims(log_pis_t, -1)
  
      # Get policy output for t+1.
      action_dist_tp1 = cls(
          fc(
              relu(
                  fc(
                      model_out_tp1,
                      weights[ks[1]],
                      weights[ks[0]],
                      framework=fw,
                  )
              ),
              weights[ks[9]],
              weights[ks[8]],
          ),
          None,
      )
      policy_tp1 = action_dist_tp1.deterministic_sample()
      log_pis_tp1 = action_dist_tp1.logp(policy_tp1)
      if sess:
          log_pis_tp1 = sess.run(log_pis_tp1)
          policy_tp1 = sess.run(policy_tp1)
      log_pis_tp1 = np.expand_dims(log_pis_tp1, -1)
  
      # Q-values for the actually selected actions.
      # get_q_values
      q_t = fc(
          relu(
              fc(
                  np.concatenate(
                      [model_out_t, train_batch[SampleBatch.ACTIONS]],
                      -1,
                  ),
                  weights[ks[3]],
                  weights[ks[2]],
                  framework=fw,
              )
          ),
          weights[ks[11]],
          weights[ks[10]],
          framework=fw,
      )
  
      # Q-values for current policy in given current state.
      # get_q_values
      q_t_det_policy = fc(
          relu(
              fc(
                  np.concatenate([model_out_t, policy_t], -1),
                  weights[ks[3]],
                  weights[ks[2]],
                  framework=fw,
              )
          ),
          weights[ks[11]],
          weights[ks[10]],
          framework=fw,
      )
  
      # Target q network evaluation.
      # target_model.get_q_values
      if fw == "tf":
          q_tp1 = fc(
              relu(
                  fc(
                      np.concatenate(
                          [target_model_out_tp1, policy_tp1], -1
                      ),
                      weights[ks[7]],
                      weights[ks[6]],
                      framework=fw,
                  )
              ),
              weights[ks[15]],
              weights[ks[14]],
              framework=fw,
          )
      else:
          assert fw == "tf2"
          q_tp1 = fc(
              relu(
                  fc(
                      np.concatenate(
                          [target_model_out_tp1, policy_tp1], -1
                      ),
                      weights[ks[7]],
                      weights[ks[6]],
                      framework=fw,
                  )
              ),
              weights[ks[9]],
              weights[ks[8]],
              framework=fw,
          )
  
      q_t_selected = np.squeeze(q_t, axis=-1)
      q_tp1 -= alpha * log_pis_tp1
      q_tp1_best = np.squeeze(q_tp1, axis=-1)
      dones = train_batch[SampleBatch.TERMINATEDS]
      rewards = train_batch[SampleBatch.REWARDS]
      if fw == "torch":
          dones = dones.float().numpy()
          rewards = rewards.numpy()
      q_tp1_best_masked = (1.0 - dones) * q_tp1_best
      q_t_selected_target = rewards + gamma * q_tp1_best_masked
      base_td_error = np.abs(q_t_selected - q_t_selected_target)
      td_error = base_td_error
      critic_loss = [
          np.mean(
              train_batch["weights"]
              * huber_loss(q_t_selected_target - q_t_selected)
          )
      ]
      target_entropy = -np.prod((1,))
      alpha_loss = -np.mean(log_alpha * (log_pis_t + target_entropy))
      actor_loss = np.mean(alpha * log_pis_t - q_t_det_policy)
  
      return critic_loss, actor_loss, alpha_loss, td_error
  
imports: import gymnasium
  from gymnasium.spaces import Box, Dict, Discrete, Tuple
  import numpy
  import re
  import unittest
  import ray
  from ray.rllib.algorithms import sac
  from ray.rllib.algorithms.sac.sac_tf_policy import sac_actor_critic_loss
  from ray.rllib.algorithms.sac.sac_torch_policy import actor_critic_loss
  from ray.rllib.examples.env.random_env import RandomEnv
  from ray.rllib.examples.models.batch_norm_model import KerasBatchNormModel, TorchBatchNormModel
  from ray.rllib.models.catalog import ModelCatalog
  from ray.rllib.models.tf.tf_action_dist import Dirichlet
  from ray.rllib.models.torch.torch_action_dist import TorchDirichlet
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.numpy import fc, huber_loss, relu
  from ray.rllib.utils.spaces.simplex import Simplex
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
  from ray.rllib.utils.torch_utils import convert_to_torch_tensor
  from ray import tune
  from ray.rllib.utils.replay_buffers.utils import patch_buffer_with_fake_sampling_method
label_desc: Use image node_id 7 for calling the TestSAC obj's underlying member method code with example usage: obj._sac_loss_helper(train_batch, weights, ks, log_alpha, fw, gamma, sess) and returns: critic_loss,  actor_loss,  alpha_loss,  td_error