class_docstr: Defines a configuration class from which a PPO Algorithm can be built.
  
  .. testcode::
  
      from ray.rllib.algorithms.ppo import PPOConfig
      config = PPOConfig()
      config = config.training(gamma=0.9, lr=0.01, kl_coeff=0.3,
          train_batch_size=128)
      config = config.resources(num_gpus=0)
      config = config.rollouts(num_rollout_workers=1)
  
      # Build a Algorithm object from the config and run 1 training iteration.
      algo = config.build(env="CartPole-v1")
      algo.train()
  
  .. testcode::
  
      from ray.rllib.algorithms.ppo import PPOConfig
      from ray import air
      from ray import tune
      config = PPOConfig()
      # Print out some default values.
  
      # Update the config object.
      config.training(
          lr=tune.grid_search([0.001 ]), clip_param=0.2
      )
      # Set the config object's env.
      config = config.environment(env="CartPole-v1")
  
      # Use to_dict() to get the old-style python config dict
      # when running with tune.
      tune.Tuner(
          "PPO",
          run_config=air.RunConfig(stop={"training_iteration": 1}),
          param_space=config.to_dict(),
      ).fit()
  
  .. testoutput::
      :hide:
  
      ...
class_docstr_tok: ["Defines","a","configuration","class","from","which","a","PPO","Algorithm","can","be","built",".","..","testcode",":",":","from","ray.rllib.algorithms.ppo","import","PPOConfig","config","=","PPOConfig","(",")","config","=","config.training","(","gamma=0.9",",","lr=0.01",",","kl_coeff=0.3",",","train_batch_size=128",")","config","=","config.resources","(","num_gpus=0",")","config","=","config.rollouts","(","num_rollout_workers=1",")","#","Build","a","Algorithm","object","from","the","config","and","run","1","training","iteration",".","algo","=","config.build","(","env=","''","CartPole-v1","''",")","algo.train","(",")","..","testcode",":",":","from","ray.rllib.algorithms.ppo","import","PPOConfig","from","ray","import","air","from","ray","import","tune","config","=","PPOConfig","(",")","#","Print","out","some","default","values",".","#","Update","the","config","object",".","config.training","(","lr=tune.grid_search","(","[","0.001","]",")",",","clip_param=0.2",")","#","Set","the","config","object","'s","env",".","config","=","config.environment","(","env=","''","CartPole-v1","''",")","#","Use","to_dict","(",")","to","get","the","old-style","python","config","dict","#","when","running","with","tune",".","tune.Tuner","(","``","PPO","''",",","run_config=air.RunConfig","(","stop=","{","``","training_iteration","''",":","1","}",")",",","param_space=config.to_dict","(",")",",",")",".fit","(",")","..","testoutput",":",":",":","hide",":","..."]
docstr: Sets the training related configuration.
  
  Args:
      lr_schedule: Learning rate schedule. In the format of
          [[timestep, lr-value], [timestep, lr-value], ...]
          Intermediary timesteps will be assigned to interpolated learning rate
          values. A schedule should normally start from timestep 0.
      use_critic: Should use a critic as a baseline (otherwise don't use value
          baseline; required for using GAE).
      use_gae: If true, use the Generalized Advantage Estimator (GAE)
          with a value function, see https://arxiv.org/pdf/1506.02438.pdf.
      lambda_: The GAE (lambda) parameter.
      use_kl_loss: Whether to use the KL-term in the loss function.
      kl_coeff: Initial coefficient for KL divergence.
      kl_target: Target value for KL divergence.
      sgd_minibatch_size: Total SGD batch size across all devices for SGD.
          This defines the minibatch size within each epoch.
      num_sgd_iter: Number of SGD iterations in each outer loop (i.e., number of
          epochs to execute per train batch).
      shuffle_sequences: Whether to shuffle sequences in the batch when training
          (recommended).
      vf_loss_coeff: Coefficient of the value function loss. IMPORTANT: you must
          tune this if you set vf_share_layers=True inside your model's config.
      entropy_coeff: Coefficient of the entropy regularizer.
      entropy_coeff_schedule: Decay schedule for the entropy regularizer.
      clip_param: The PPO clip parameter.
      vf_clip_param: Clip param for the value function. Note that this is
          sensitive to the scale of the rewards. If your expected V is large,
          increase this.
      grad_clip: If specified, clip the global norm of gradients by this amount.
  
  Returns:
      This updated AlgorithmConfig object.
docstr_tok: ["Sets","the","training","related","configuration",".","Args",":","lr_schedule",":","Learning","rate","schedule",".","In","the","format","of","[","[","timestep",",","lr-value","]",",","[","timestep",",","lr-value","]",",","...","]","Intermediary","timesteps","will","be","assigned","to","interpolated","learning","rate","values",".","A","schedule","should","normally","start","from","timestep","0.","use_critic",":","Should","use","a","critic","as","a","baseline","(","otherwise","do","n't","use","value","baseline",";","required","for","using","GAE",")",".","use_gae",":","If","true",",","use","the","Generalized","Advantage","Estimator","(","GAE",")","with","a","value","function",",","see","https",":","\/\/arxiv.org\/pdf\/1506.02438.pdf",".","lambda_",":","The","GAE","(","lambda",")","parameter",".","use_kl_loss",":","Whether","to","use","the","KL-term","in","the","loss","function",".","kl_coeff",":","Initial","coefficient","for","KL","divergence",".","kl_target",":","Target","value","for","KL","divergence",".","sgd_minibatch_size",":","Total","SGD","batch","size","across","all","devices","for","SGD",".","This","defines","the","minibatch","size","within","each","epoch",".","num_sgd_iter",":","Number","of","SGD","iterations","in","each","outer","loop","(","i.e.",",","number","of","epochs","to","execute","per","train","batch",")",".","shuffle_sequences",":","Whether","to","shuffle","sequences","in","the","batch","when","training","(","recommended",")",".","vf_loss_coeff",":","Coefficient","of","the","value","function","loss",".","IMPORTANT",":","you","must","tune","this","if","you","set","vf_share_layers=True","inside","your","model","'s","config",".","entropy_coeff",":","Coefficient","of","the","entropy","regularizer",".","entropy_coeff_schedule",":","Decay","schedule","for","the","entropy","regularizer",".","clip_param",":","The","PPO","clip","parameter",".","vf_clip_param",":","Clip","param","for","the","value","function",".","Note","that","this","is","sensitive","to","the","scale","of","the","rewards",".","If","your","expected","V","is","large",",","increase","this",".","grad_clip",":","If","specified",",","clip","the","global","norm","of","gradients","by","this","amount",".","Returns",":","This","updated","AlgorithmConfig","object","."]
code: def training(
      self,
      *,
      lr_schedule: Optional[
          List[List[Union[int, float]]]
      ] = NotProvided,
      use_critic: Optional[bool] = NotProvided,
      use_gae: Optional[bool] = NotProvided,
      lambda_: Optional[float] = NotProvided,
      use_kl_loss: Optional[bool] = NotProvided,
      kl_coeff: Optional[float] = NotProvided,
      kl_target: Optional[float] = NotProvided,
      sgd_minibatch_size: Optional[int] = NotProvided,
      num_sgd_iter: Optional[int] = NotProvided,
      shuffle_sequences: Optional[bool] = NotProvided,
      vf_loss_coeff: Optional[float] = NotProvided,
      entropy_coeff: Optional[float] = NotProvided,
      entropy_coeff_schedule: Optional[
          List[List[Union[int, float]]]
      ] = NotProvided,
      clip_param: Optional[float] = NotProvided,
      vf_clip_param: Optional[float] = NotProvided,
      grad_clip: Optional[float] = NotProvided,
      # Deprecated.
      vf_share_layers=DEPRECATED_VALUE,
      **kwargs,
  ) -> "PPOConfig":
      """Sets the training related configuration.
  
      Args:
          lr_schedule: Learning rate schedule. In the format of
              [[timestep, lr-value], [timestep, lr-value], ...]
              Intermediary timesteps will be assigned to interpolated learning rate
              values. A schedule should normally start from timestep 0.
          use_critic: Should use a critic as a baseline (otherwise don't use value
              baseline; required for using GAE).
          use_gae: If true, use the Generalized Advantage Estimator (GAE)
              with a value function, see https://arxiv.org/pdf/1506.02438.pdf.
          lambda_: The GAE (lambda) parameter.
          use_kl_loss: Whether to use the KL-term in the loss function.
          kl_coeff: Initial coefficient for KL divergence.
          kl_target: Target value for KL divergence.
          sgd_minibatch_size: Total SGD batch size across all devices for SGD.
              This defines the minibatch size within each epoch.
          num_sgd_iter: Number of SGD iterations in each outer loop (i.e., number of
              epochs to execute per train batch).
          shuffle_sequences: Whether to shuffle sequences in the batch when training
              (recommended).
          vf_loss_coeff: Coefficient of the value function loss. IMPORTANT: you must
              tune this if you set vf_share_layers=True inside your model's config.
          entropy_coeff: Coefficient of the entropy regularizer.
          entropy_coeff_schedule: Decay schedule for the entropy regularizer.
          clip_param: The PPO clip parameter.
          vf_clip_param: Clip param for the value function. Note that this is
              sensitive to the scale of the rewards. If your expected V is large,
              increase this.
          grad_clip: If specified, clip the global norm of gradients by this amount.
  
      Returns:
          This updated AlgorithmConfig object.
      """
      # Pass kwargs onto super's `training()` method.
      super().training(**kwargs)
  
      if use_critic is not NotProvided:
          self.use_critic = use_critic
          # TODO (Kourosh) This is experimental.
          #  Don't forget to remove .use_critic from algorithm config.
      if use_gae is not NotProvided:
          self.use_gae = use_gae
      if lambda_ is not NotProvided:
          self.lambda_ = lambda_
      if use_kl_loss is not NotProvided:
          self.use_kl_loss = use_kl_loss
      if kl_coeff is not NotProvided:
          self.kl_coeff = kl_coeff
      if kl_target is not NotProvided:
          self.kl_target = kl_target
      if sgd_minibatch_size is not NotProvided:
          self.sgd_minibatch_size = sgd_minibatch_size
      if num_sgd_iter is not NotProvided:
          self.num_sgd_iter = num_sgd_iter
      if shuffle_sequences is not NotProvided:
          self.shuffle_sequences = shuffle_sequences
      if vf_loss_coeff is not NotProvided:
          self.vf_loss_coeff = vf_loss_coeff
      if entropy_coeff is not NotProvided:
          self.entropy_coeff = entropy_coeff
      if clip_param is not NotProvided:
          self.clip_param = clip_param
      if vf_clip_param is not NotProvided:
          self.vf_clip_param = vf_clip_param
      if grad_clip is not NotProvided:
          self.grad_clip = grad_clip
  
      # TODO (sven): Remove these once new API stack is only option for PPO.
      if lr_schedule is not NotProvided:
          self.lr_schedule = lr_schedule
      if entropy_coeff_schedule is not NotProvided:
          self.entropy_coeff_schedule = entropy_coeff_schedule
  
      return self
  
imports: import logging
  from typing import List, Optional, Type, Union, TYPE_CHECKING
  import numpy
  import tree
  from ray.rllib.algorithms.algorithm import Algorithm
  from ray.rllib.algorithms.algorithm_config import AlgorithmConfig, NotProvided
  from ray.rllib.core.rl_module.rl_module import SingleAgentRLModuleSpec
  from ray.rllib.evaluation.postprocessing_v2 import postprocess_episodes_to_sample_batch
  from ray.rllib.evaluation.rollout_worker import RolloutWorker
  from ray.rllib.execution.rollout_ops import standardize_fields, synchronous_parallel_sample
  from ray.rllib.execution.train_ops import train_one_step, multi_gpu_train_one_step
  from ray.rllib.policy.policy import Policy
  from ray.rllib.utils.annotations import override
  from ray.rllib.utils.deprecation import DEPRECATED_VALUE
  from ray.rllib.utils.metrics.learner_info import LEARNER_STATS_KEY
  from ray.rllib.utils.metrics import NUM_AGENT_STEPS_SAMPLED, NUM_ENV_STEPS_SAMPLED, SYNCH_WORKER_WEIGHTS_TIMER, SAMPLE_TIMER, ALL_MODULES
  from ray.rllib.env.single_agent_episode import SingleAgentEpisode
  from ray.rllib.policy.sample_batch import SampleBatch
  from ray.rllib.utils.schedules.scheduler import Scheduler
  from ray.rllib.utils.typing import ResultDict
  from ray.util.debug import log_once
label_desc: Use image node_id 4 for calling the PPOConfig obj's underlying member method code with example usage: obj.training() and returns: self