class_docstr: Model for DPO language modeling problem type.
docstr: Forward pass of DPO model.
  Runtime is 4 times slower than causal language modeling model
  as we need to compute
  - logits for chosen answer
  - logits for rejected answer
  - logits for chosen answer with reference model
  - logits for rejected answer with reference model
docstr_tok: ["Forward","pass","of","DPO","model",".","Runtime","is","4","times","slower","than","causal","language","modeling","model","as","we","need","to","compute","-","logits","for","chosen","answer","-","logits","for","rejected","answer","-","logits","for","chosen","answer","with","reference","model","-","logits","for","rejected","answer","with","reference","model"]
code: def forward(
      self,
      batch: Dict,
      padding: bool = True,
  ) -> Dict:
      """
      Forward pass of DPO model.
      Runtime is 4 times slower than causal language modeling model
      as we need to compute
      - logits for chosen answer
      - logits for rejected answer
      - logits for chosen answer with reference model
      - logits for rejected answer with reference model
      """
      # disable cache if gradient checkpointing is enabled
      if self.cfg.architecture.gradient_checkpointing:
          self.backbone.config.use_cache = False
  
      outputs: Dict = {}
  
      chosen_logits = None
      chosen_labels = None
  
      for answer in ["chosen", "rejected"]:
          if padding:
              batch = batch_padding(
                  self.cfg,
                  batch,
                  self.training,
                  mask_key=f"{answer}_attention_mask",
                  pad_keys=[
                      f"{answer}_input_ids",
                      f"{answer}_attention_mask",
                      f"{answer}_labels",
                  ],
              )
          logits = self.backbone(
              input_ids=batch[f"{answer}_input_ids"],
              attention_mask=batch[f"{answer}_attention_mask"],
          ).logits
          chosen_logits = (
              logits.detach() if answer == "chosen" else chosen_logits
          )
          chosen_labels = (
              batch[f"{answer}_labels"]
              if answer == "chosen"
              else chosen_labels
          )
  
          outputs[f"{answer}_logps"] = get_batch_logps(
              logits, batch[f"{answer}_labels"]
          )
  
          with self.backbone.disable_adapter():
              with torch.no_grad():
                  reference_logits = self.backbone(
                      input_ids=batch[f"{answer}_input_ids"],
                      attention_mask=batch[f"{answer}_attention_mask"],
                  ).logits
                  outputs[
                      f"{answer}_reference_logps"
                  ] = get_batch_logps(
                      reference_logits, batch[f"{answer}_labels"]
                  )
  
      loss, chosen_rewards, rejected_rewards = self.loss_fn(
          policy_chosen_logps=outputs["chosen_logps"],
          policy_rejected_logps=outputs["rejected_logps"],
          reference_chosen_logps=outputs["chosen_reference_logps"],
          reference_rejected_logps=outputs["rejected_reference_logps"],
      )
      outputs["loss"] = loss
  
      # These values will be logged to Neptune, if enabled, see train.py
      outputs["chosen_rewards"] = chosen_rewards
      outputs["rejected_rewards"] = rejected_rewards
      # Reward margin should increase over time
      outputs["reward_margin"] = chosen_rewards - rejected_rewards
  
      if (
          not self.training
          and self.cfg.prediction.metric == "Perplexity"
      ):
          outputs["perplexity"] = self.perplexity(
              chosen_logits, chosen_labels
          )
  
      # enable cache again if gradient checkpointing is enabled
      if self.cfg.architecture.gradient_checkpointing:
          self.backbone.config.use_cache = True
  
      return outputs
  
file_path: h2o-llmstudio/llm_studio/src/models/text_dpo_modeling_model.py
imports: import logging
  from typing import Any, Dict
  import torch
  from torch import nn
  from transformers import AutoModelForCausalLM
  from llm_studio.src.metrics.text_causal_language_modeling_metrics import Perplexity
  from llm_studio.src.utils.data_utils import batch_padding
  from llm_studio.src.utils.modeling_utils import create_nlp_backbone, generate, prepare_lora
label_desc: Use image node_id 3 for calling the Model obj's underlying member method code with example usage: obj.forward(batch, padding) and returns: outputs