class_docstr: The PPOTrainer uses Proximal Policy Optimization to optimise language models.
  Note, this trainer is heavily inspired by the original OpenAI learning to summarize
  work here: https://github.com/openai/summarize-from-feedback
  
  Attributes:
      **cfg** (`LLM Studio Config`) -- Experiment configuration object. Check the
          documentation of `LLM Studio Config` for more details.
      **model** (`PreTrainedModelWrapper`) -- Model to be optimized, Hugging Face
          transformer model with a value head. Check the documentation of
          `PreTrainedModelWrapper` for more details.
      **tokenizer** (`Union[PreTrainedTokenizer, PreTrainedTokenizerFast]`)
          Tokenizer to be used for encoding the data. Check the documentation of
          `transformers.PreTrainedTokenizer` and
          `transformers.PreTrainedTokenizerFast` for more details.
      **optimizer** (`torch.optim.Optimizer`) -- Optimizer to be used for training.
      **lr_scheduler** (`torch.optim.lr_scheduler`) -- Learning rate scheduler to be
          used for training.
      **scaler** (`torch.cuda.amp.GradScaler`) -- Gradient scaler to be used for
          training.
class_docstr_tok: ["The","PPOTrainer","uses","Proximal","Policy","Optimization","to","optimise","language","models",".","Note",",","this","trainer","is","heavily","inspired","by","the","original","OpenAI","learning","to","summarize","work","here",":","https",":","\/\/github.com\/openai\/summarize-from-feedback","Attributes",":","*","*","cfg","*","*","(","`","LLM","Studio","Config","`",")","--","Experiment","configuration","object",".","Check","the","documentation","of","`","LLM","Studio","Config","`","for","more","details",".","*","*","model","*","*","(","`","PreTrainedModelWrapper","`",")","--","Model","to","be","optimized",",","Hugging","Face","transformer","model","with","a","value","head",".","Check","the","documentation","of","`","PreTrainedModelWrapper","`","for","more","details",".","*","*","tokenizer","*","*","(","`","Union","[","PreTrainedTokenizer",",","PreTrainedTokenizerFast","]","`",")","Tokenizer","to","be","used","for","encoding","the","data",".","Check","the","documentation","of","`","transformers.PreTrainedTokenizer","`","and","`","transformers.PreTrainedTokenizerFast","`","for","more","details",".","*","*","optimizer","*","*","(","`","torch.optim.Optimizer","`",")","--","Optimizer","to","be","used","for","training",".","*","*","lr_scheduler","*","*","(","`","torch.optim.lr_scheduler","`",")","--","Learning","rate","scheduler","to","be","used","for","training",".","*","*","scaler","*","*","(","`","torch.cuda.amp.GradScaler","`",")","--","Gradient","scaler","to","be","used","for","training","."]
docstr: Initialize PPOTrainer.
  
  Args:
      cfg (`LLM Studio Config`):
          experiment configuration object. Check the documentation of
          `LLM Studio Config` for more details.
      model (`PreTrainedModelWrapper`):
          Hugging Face transformer model with a value head.
      tokenizer (`transformers.PreTrainedTokenizer`):
          Hugging Face tokenizer
      optimizer (`torch.optim.Optimizer`):
          Optimizer used for training.
      lr_scheduler (`torch.optim.lr_scheduler`):
          Learning rate scheduler used for training.
      scaler (`torch.cuda.amp.GradScaler`):
          Gradient scaler used for training.
code: def __init__(
      self,
      cfg=None,
      model=None,
      tokenizer: Union[
          PreTrainedTokenizer, PreTrainedTokenizerFast
      ] = None,
      optimizer: Optional[torch.optim.Optimizer] = None,
      lr_scheduler: Optional[
          torch.optim.lr_scheduler._LRScheduler
      ] = None,
      scaler=None,
  ):
      """
      Initialize PPOTrainer.
  
      Args:
          cfg (`LLM Studio Config`):
              experiment configuration object. Check the documentation of
              `LLM Studio Config` for more details.
          model (`PreTrainedModelWrapper`):
              Hugging Face transformer model with a value head.
          tokenizer (`transformers.PreTrainedTokenizer`):
              Hugging Face tokenizer
          optimizer (`torch.optim.Optimizer`):
              Optimizer used for training.
          lr_scheduler (`torch.optim.lr_scheduler`):
              Learning rate scheduler used for training.
          scaler (`torch.cuda.amp.GradScaler`):
              Gradient scaler used for training.
      """
      self.cfg = cfg
  
      # Step 1: Initialize Model
      self.model = model
      self.tokenizer = tokenizer
  
      # Step 3: Initialize optimizer and data collator
      self.data_collator = DataCollatorForLanguageModeling(
          self.tokenizer, mlm=False
      )
      self.optimizer = optimizer
      self.lr_scheduler = lr_scheduler
      self.scaler = scaler
      self.num_updates = 0
  
      self.kl_ctl: AdaptiveKLController | FixedKLController
      if self.cfg.training.adaptive_kl_control:
          self.kl_ctl = AdaptiveKLController(cfg)
      else:
          self.kl_ctl = FixedKLController(cfg)
  
      self.current_device = self.cfg.environment._device
  
      # init the current step
      self.current_step = 0
  
imports: import math
  import time
  import warnings
  from collections.abc import Mapping
  from typing import Any, Dict, List, Optional, Union
  import numpy
  import torch
  import torch.nn.functional
  from datasets import Dataset
  from huggingface_hub import PyTorchModelHubMixin
  from torch.cuda.amp import autocast
  from torch.nn.utils.rnn import pad_sequence
  from torch.utils.data import DataLoader
  from transformers import DataCollatorForLanguageModeling, PreTrainedTokenizer, PreTrainedTokenizerFast
  from llm_studio.src.utils.modeling_utils import unwrap_model
label_desc: Use image node_id 1 to create a new PPOTrainer object from inherited base classes: PyTorchModelHubMixin with example: obj = PPOTrainer(cfg, model, tokenizer, optimizer, lr_scheduler, scaler)