class_docstr: Support the CLIP model.
  Refer to https://huggingface.co/docs/transformers/model_doc/clip
class_docstr_tok: ["Support","the","CLIP","model",".","Refer","to","https",":","\/\/huggingface.co\/docs\/transformers\/model_doc\/clip"]
docstr: Parameters
  ----------
  batch
      A dictionary containing the input mini-batch data.
      We need to use the keys with the model prefix to index required data.
  
  Returns
  -------
      A dictionary with logits and features.
docstr_tok: ["Parameters","--","--","--","--","--","batch","A","dictionary","containing","the","input","mini-batch","data",".","We","need","to","use","the","keys","with","the","model","prefix","to","index","required","data",".","Returns","--","--","--","-","A","dictionary","with","logits","and","features","."]
code: def forward(
      self,
      batch: dict,
  ):
      """
      Parameters
      ----------
      batch
          A dictionary containing the input mini-batch data.
          We need to use the keys with the model prefix to index required data.
  
      Returns
      -------
          A dictionary with logits and features.
      """
      has_image = self.image_key in batch
      has_text = self.text_token_ids_key in batch
      ret = {COLUMN_FEATURES: {FEATURES: {}, MASKS: {}}}
  
      if has_image:
          images = batch[self.image_key]
          image_valid_num = batch[self.image_valid_num_key]
          assert images.dim() == 5
          b, n, c, h, w = images.shape
          vision_outputs = self.model.vision_model(
              pixel_values=images.reshape((b * n, c, h, w)),
              output_attentions=True,
              output_hidden_states=True,
              return_dict=True,
          )
          image_features = self.model.visual_projection(
              vision_outputs.pooler_output
          )
          steps = torch.arange(0, n).type_as(image_valid_num)
          image_masks = (
              steps.reshape((1, -1)) < image_valid_num.reshape((-1, 1))
          ).type_as(
              image_features
          )  # (b, n)
          image_features = (
              image_features.reshape((b, n, -1))
              * image_masks[:, :, None]
          )  # (b, n, num_features)
  
          # normalized features
          image_features = image_features / image_features.norm(
              dim=-1, keepdim=True
          )
  
          # collect image features by image column names
          (
              image_column_features,
              image_column_feature_masks,
          ) = get_column_features(
              batch=batch,
              column_name_prefix=self.image_column_prefix,
              features=image_features,
              valid_lengths=image_valid_num,
          )
          ret[COLUMN_FEATURES][FEATURES].update(image_column_features)
          ret[COLUMN_FEATURES][MASKS].update(image_column_feature_masks)
  
          image_features = image_features.mean(
              dim=1
          )  # (b, num_features)
          ret[FEATURES] = image_features
  
      if has_text:
          text_token_ids = batch[self.text_token_ids_key]
          text_valid_length = batch[self.text_valid_length_key]
          steps = torch.arange(0, text_token_ids.shape[1]).type_as(
              text_valid_length
          )
          text_masks = (
              steps.reshape((1, -1))
              < text_valid_length.reshape((-1, 1))
          ).type_as(text_token_ids)
          assert torch.equal(text_valid_length, text_masks.sum(dim=-1))
  
          text_outputs = self.model.text_model(
              input_ids=text_token_ids,
              attention_mask=text_masks,
              output_attentions=True,
              output_hidden_states=True,
              return_dict=True,
          )
          text_features = self.model.text_projection(
              text_outputs.pooler_output
          )  # (b, num_features)
  
          # normalized features
          text_features = text_features / text_features.norm(
              dim=-1, keepdim=True
          )
  
          # collect text features by text column names
          (
              text_column_features,
              text_column_feature_masks,
          ) = get_column_features(
              batch=batch,
              column_name_prefix=self.text_column_prefix,
              features=self.model.text_projection(
                  text_outputs.last_hidden_state
              ),
              valid_lengths=text_valid_length,
              cls_feature=text_features,
          )
          ret[COLUMN_FEATURES][FEATURES].update(text_column_features)
          ret[COLUMN_FEATURES][MASKS].update(text_column_feature_masks)
          ret[FEATURES] = text_features
  
      if has_image and has_text:
          if self.num_classes:
              features = image_features + text_features
              logits = self.head(features)
              ret[FEATURES] = features
          else:
              # cosine similarity as logits
              logits = torch.sum(image_features * text_features, dim=-1)
  
          ret[LOGITS] = logits
  
      ret[LOGIT_SCALE] = self.model.logit_scale.exp()
  
      return {self.prefix: ret}
  
file_path: autogluon/multimodal/src/autogluon/multimodal/models/clip.py
imports: import logging
  from typing import Optional
  import torch
  from torch import nn
  from ..constants import AUTOMM, COLUMN, COLUMN_FEATURES, FEATURES, IMAGE, IMAGE_VALID_NUM, LABEL, LOGIT_SCALE, LOGITS, MASKS, TEXT_TOKEN_IDS, TEXT_VALID_LENGTH
  from .utils import assign_layer_ids, get_column_features, get_hf_config_and_model, get_pretrained_tokenizer, init_weights
label_desc: Use image node_id 11 for calling the CLIPForImageText obj's underlying member method code with example usage: obj.forward(batch) and returns: dict