hunk.code_window:             train_batch[Postprocessing.ADVANTAGES]
               * torch.clamp(
                   logp_ratio, 1 - self.config["clip_param"], 1 + self.config["clip_param"]
               ),
           )
           mean_policy_loss = reduce_mean_valid(-surrogate_loss)
           # Compute a value function loss.
           if self.config["use_critic"]:
               value_fn_out = model.value_function()
  
hunk.labels: keep keep keep keep keep replace keep keep keep keep
hunk.file_path: rllib/algorithms/ppo/ppo_torch_policy.py
hunk.type: replace
file: base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }}
  env_vars: {"HOROVOD_GLOO_TIMEOUT_SECONDS": "120"}
  debian_packages:
    - curl
  
  python:
    pip_packages:
      - pytest
      - awscli
    conda_packages: []
  
  post_build_cmds:
    - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
    - pip3 install 'ray[tune]'
    - pip3 install torch torchvision
    - HOROVOD_WITH_GLOO=1 HOROVOD_WITHOUT_MPI=1 HOROVOD_WITHOUT_TENSORFLOW=1 HOROVOD_WITHOUT_MXNET=1 HOROVOD_WITH_PYTORCH=1 pip3 install -U horovod
    - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
  