docstr: To generate the historic data used in this test case, first run:
  $ ./train.py --run=PPO --env=CartPole-v1           --stop='{"timesteps_total": 50000}'           --config='{"output": "/tmp/out", "batch_mode": "complete_episodes"}'
docstr_tok: ["To","generate","the","historic","data","used","in","this","test","case",",","first","run",":","$",".\/train.py","--","run=PPO","--","env=CartPole-v1","--","stop=","'","{","``","timesteps_total","''",":","50000","}","'","--","config=","'","{","``","output","''",":","``","\/tmp\/out","''",",","``","batch_mode","''",":","``","complete_episodes","''","}","'"]
code: def test_marwil_loss_function(self):
      """
          To generate the historic data used in this test case, first run:
          $ ./train.py --run=PPO --env=CartPole-v1 \
            --stop='{"timesteps_total": 50000}' \
            --config='{"output": "/tmp/out", "batch_mode": "complete_episodes"}'
          """
      rllib_dir = Path(__file__).parent.parent.parent.parent
      print("rllib dir={}".format(rllib_dir))
      data_file = os.path.join(
          rllib_dir, "tests/data/cartpole/small.json"
      )
      print(
          "data_file={} exists={}".format(
              data_file, os.path.isfile(data_file)
          )
      )
  
      config = (
          marwil.MARWILConfig()
          .rollouts(num_rollout_workers=0)
          .offline_data(input_=[data_file])
      )  # Learn from offline data.
  
      for fw, sess in framework_iterator(config, session=True):
          reader = JsonReader(inputs=[data_file])
          batch = reader.next()
  
          algo = config.build(env="CartPole-v1")
          policy = algo.get_policy()
          model = policy.model
  
          # Calculate our own expected values (to then compare against the
          # agent's loss output).
          cummulative_rewards = compute_advantages(
              batch, 0.0, config.gamma, 1.0, False, False
          )["advantages"]
          if fw == "torch":
              cummulative_rewards = torch.tensor(cummulative_rewards)
          if fw != "tf":
              batch = policy._lazy_tensor_dict(batch)
          model_out, _ = model(batch)
          vf_estimates = model.value_function()
          if fw == "tf":
              model_out, vf_estimates = policy.get_session().run(
                  [model_out, vf_estimates]
              )
          adv = cummulative_rewards - vf_estimates
          if fw == "torch":
              adv = adv.detach().cpu().numpy()
          adv_squared = np.mean(np.square(adv))
          c_2 = 100.0 + 1e-8 * (adv_squared - 100.0)
          c = np.sqrt(c_2)
          exp_advs = np.exp(config.beta * (adv / c))
          dist = policy.dist_class(model_out, model)
          logp = dist.logp(batch["actions"])
          if fw == "torch":
              logp = logp.detach().cpu().numpy()
          elif fw == "tf":
              logp = sess.run(logp)
          # Calculate all expected loss components.
          expected_vf_loss = 0.5 * adv_squared
          expected_pol_loss = -1.0 * np.mean(exp_advs * logp)
          expected_loss = (
              expected_pol_loss + config.vf_coeff * expected_vf_loss
          )
  
          # Calculate the algorithm's loss (to check against our own
          # calculation above).
          batch.set_get_interceptor(None)
          postprocessed_batch = policy.postprocess_trajectory(batch)
          loss_func = (
              MARWILTF2Policy.loss
              if fw != "torch"
              else MARWILTorchPolicy.loss
          )
          if fw != "tf":
              policy._lazy_tensor_dict(postprocessed_batch)
              loss_out = loss_func(
                  policy, model, policy.dist_class, postprocessed_batch
              )
          else:
              loss_out, v_loss, p_loss = policy.get_session().run(
                  # policy._loss is create by TFPolicy, and is basically the
                  # loss tensor of the static graph.
                  [
                      policy._loss,
                      policy._marwil_loss.v_loss,
                      policy._marwil_loss.p_loss,
                  ],
                  feed_dict=policy._get_loss_inputs_dict(
                      postprocessed_batch, shuffle=False
                  ),
              )
  
          # Check all components.
          if fw == "torch":
              check(policy.v_loss, expected_vf_loss, decimals=4)
              check(policy.p_loss, expected_pol_loss, decimals=4)
          elif fw == "tf":
              check(v_loss, expected_vf_loss, decimals=4)
              check(p_loss, expected_pol_loss, decimals=4)
          else:
              check(
                  policy._marwil_loss.v_loss,
                  expected_vf_loss,
                  decimals=4,
              )
              check(
                  policy._marwil_loss.p_loss,
                  expected_pol_loss,
                  decimals=4,
              )
          check(loss_out, expected_loss, decimals=3)
  
file_path: ray/python/ray/rllib/algorithms/marwil/tests/test_marwil.py
imports: import numpy
  import os
  from pathlib import Path
  import unittest
  import ray
  import ray.rllib.algorithms.marwil
  from ray.rllib.algorithms.marwil.marwil_tf_policy import MARWILTF2Policy
  from ray.rllib.algorithms.marwil.marwil_torch_policy import MARWILTorchPolicy
  from ray.rllib.evaluation.postprocessing import compute_advantages
  from ray.rllib.offline import JsonReader
  from ray.rllib.utils.framework import try_import_tf, try_import_torch
  from ray.rllib.utils.test_utils import check, check_compute_single_action, check_train_results, framework_iterator
label_desc: Use image node_id 5 for calling the TestMARWIL obj's underlying member method code with example usage: obj.test_marwil_loss_function() without return types