class_docstr: Support the OVD models.
  Now support GroundingDINO: https://github.com/IDEA-Research/GroundingDINO
class_docstr_tok: ["Support","the","OVD","models",".","Now","support","GroundingDINO",":","https",":","\/\/github.com\/IDEA-Research\/GroundingDINO"]
docstr: Parameters
  ----------
  batch
      A dictionary containing the input mini-batch data.
      We need to use the keys with the model prefix to index required data.
  
  Returns
  -------
      A dictionary with logits and features.
docstr_tok: ["Parameters","--","--","--","--","--","batch","A","dictionary","containing","the","input","mini-batch","data",".","We","need","to","use","the","keys","with","the","model","prefix","to","index","required","data",".","Returns","--","--","--","-","A","dictionary","with","logits","and","features","."]
code: def forward(
      self,
      batch: dict,
      with_logits: bool = False,
  ):
      """
      Parameters
      ----------
      batch
          A dictionary containing the input mini-batch data.
          We need to use the keys with the model prefix to index required data.
  
      Returns
      -------
          A dictionary with logits and features.
      """
      from groundingdino.util.utils import get_phrases_from_posmap
  
      def process_caption(caption):  # TODO: split if it's too long
          caption = caption.lower().strip()
          if not caption.endswith("."):
              caption = caption + "."
          return caption
  
      captions = [
          process_caption(caption) for caption in batch[self.prompt_key]
      ]
      images = batch[self.image_key]
  
      with torch.no_grad():
          outputs = self.model(images, captions=captions)
      logits = (
          outputs["pred_logits"].cpu().sigmoid()
      )  # (bs, nq, 256)  # TODO: will .cpu() affect the data collection of lightning once we support multi gpu?
      boxes = outputs["pred_boxes"].cpu()  # (bs, nq, 4)
  
      # filter output
      batch_size = logits.shape[0]  # bs
      logits_filt = logits.clone()  # bs, num_filt(900), 256
      boxes_filt = boxes.clone()  # bs, num_filt(900), 4
      filt_mask = (
          logits_filt.max(dim=2)[0] > self.box_threshold
      )  # bs, num_filt(900)
      logits_filt = [
          logits_filt[sample_idx][filt_mask[sample_idx]]
          for sample_idx in range(batch_size)
      ]  # [bs * (num_after_mask_this_sample, 256)]
      boxes_filt = [
          boxes_filt[sample_idx][filt_mask[sample_idx]]
          for sample_idx in range(batch_size)
      ]  # [bs * (num_after_mask_this_sample, 4)]
  
      # get phrase
      tokenizer = self.model.tokenizer
      tokenized = [tokenizer(caption) for caption in captions]
      # build pred
      pred_phrases = []
      for sample_idx in range(batch_size):
          pred_phrase_per_sample = []
          logits_filt_per_sample = logits_filt[sample_idx]
          boxes_filt_per_sample = boxes_filt[sample_idx]
          tokenized_per_sample = tokenized[sample_idx]
          for logit, box in zip(
              logits_filt_per_sample, boxes_filt_per_sample
          ):
              pred_phrase = get_phrases_from_posmap(
                  logit > self.text_threshold,
                  tokenized_per_sample,
                  tokenizer,
              )
              if with_logits:
                  pred_phrase_per_sample.append(
                      pred_phrase + f"({str(logit.max().item())[:4]})"
                  )
              else:
                  pred_phrase_per_sample.append(pred_phrase)
          pred_phrases.append(pred_phrase_per_sample)
  
      ret = {
          BBOX: boxes_filt,
          PROMPT: pred_phrases,
          LOGITS: [
              logits_filt_per_sample.max(dim=1)[0]
              for logits_filt_per_sample in logits_filt
          ],
          IMAGE_META: batch[self.image_meta_key],
      }
  
      return {self.prefix: ret}
  
file_path: autogluon/multimodal/src/autogluon/multimodal/models/ovd.py
imports: import logging
  import os
  from typing import Optional
  import torch
  from torch import nn
  from ..constants import AUTOMM, BBOX, COLUMN, COLUMN_FEATURES, FEATURES, IMAGE, IMAGE_META, IMAGE_VALID_NUM, LABEL, LOGIT_SCALE, LOGITS, MASKS, PROMPT, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH
  from .utils import assign_layer_ids, get_column_features, init_weights
label_desc: Use image node_id 9 for calling the OVDModel obj's underlying member method code with example usage: obj.forward(batch, with_logits) and returns: dict, caption