code: def forward(self, samples, is_train=True):
      visual_inputs = samples["video"]
      question = samples["text_input"]
      targets = samples["answers"]
  
      # forward text
      text = self.tokenizer(
          question,
          padding="max_length",
          truncation=True,
          max_length=self.max_txt_len,
          return_tensors="pt",
      ).to(self.device)
  
      text_output = self.text_encoder.forward_text(
          text,
          token_type_ids=torch.zeros(
              text.input_ids.shape, dtype=torch.long, device=self.device
          ),
      )
      text_embeds = text_output.last_hidden_state
  
      # forward visual
      # timeSformer asks for (b, c, t, h, w) as input.
      video_embeds = self.visual_encoder.forward_features(visual_inputs)
      video_atts = torch.ones(
          video_embeds.size()[:-1], dtype=torch.long
      ).to(self.device)
  
      # forward cross-encoder
      attention_mask = torch.cat(
          [text.attention_mask, video_atts], dim=1
      )
      embedding_output = torch.cat([text_embeds, video_embeds], dim=1)
  
      encoder_output = self.text_encoder(
          encoder_embeds=embedding_output,
          attention_mask=attention_mask,
          return_dict=True,
          mode="fusion",
      )
  
      prediction = self.classifier(
          encoder_output.last_hidden_state[:, 0, :]
      )
      if is_train:
          loss = F.cross_entropy(prediction, targets)
          # return {"loss": loss}
          return AlproOutputWithLogits(
              loss=loss,
              intermediate_output=AlproIntermediateOutput(
                  video_embeds=video_embeds,
                  text_embeds=text_embeds,
                  encoder_output=encoder_output,
              ),
              logits=prediction,
          )
      else:
          return {"predictions": prediction, "targets": targets}
  
file_path: lavis/lavis/models/alpro_models/alpro_qa.py
imports: from warnings import warn
  import torch
  import torch.nn.functional
  from lavis.common.config import node_to_dict
  from lavis.common.registry import registry
  from lavis.models.alpro_models import AlproBase
  from lavis.models.alpro_models.alpro_outputs import AlproIntermediateOutput, AlproOutputWithLogits
  from lavis.models.med import XBertEncoder
  from lavis.models.timesformer.vit import TimeSformer
  from torch import nn
label_desc: Use image node_id 2 for calling the AlproQA obj's underlying member method code with example usage: obj.forward(samples, is_train) and returns: AlproOutputWithLogits, dict