class_docstr: BLIP2 Vicuna model.
  Supported model types:
      - vicuna7b
      - vicuna13b
  Usage:
      >>> from lavis.models import load_model
      >>> model = load_model("blip2_vicuna_xinstruct", "vicuna7b")
class_docstr_tok: ["BLIP2","Vicuna","model",".","Supported","model","types",":","-","vicuna7b","-","vicuna13b","Usage",":",">",">",">","from","lavis.models","import","load_model",">",">",">","model","=","load_model","(","``","blip2_vicuna_xinstruct","''",",","``","vicuna7b","''",")"]
code: def _predict_class(
      self,
      samples,
      candidates,
      n_segments=1,
      special_qformer_input_prompt=False,
  ):
      if list(samples.keys()) == []:
          return None
  
      if "prompt" in samples:
          prompt = samples["prompt"]
      else:
          prompt = self.prompt
  
      candidates = [
          self.format_candidates_prompt.format(c) for c in candidates
      ]
  
      if "modalities" in samples:
          curr_modalities = (
              samples["modalities"][0]
              if isinstance(samples["modalities"][0], list)
              else samples["modalities"]
          )
      else:
          curr_modalities = [
              modality
              for modality in self.modalities
              if modality in samples
          ]
  
      # get batch size
      for modality in curr_modalities:
          data = samples[modality]
          if isinstance(data, torch.Tensor):
              bs = data.size(0)
          else:
              bs = len(data)
          break
  
      if isinstance(prompt, str):
          prompt = [prompt] * bs
      else:
          assert (
              len(prompt) == bs
          ), "The number of prompts must be equal to the batch size."
  
      if "text_input" in samples.keys():
          if type(samples["text_input"][0]) == list:
              prompt = [
                  prompt[i].format(*samples["text_input"][i])
                  for i in range(len(prompt))
              ]
          else:
              prompt = [
                  prompt[i].format(samples["text_input"][i])
                  for i in range(len(prompt))
              ]
  
      # scienceqa
      if "context" in samples.keys() and samples["context"] != "":
          prompt = [
              f'context: {samples["context"][i]}. {prompt[i]}'
              for i in range(len(prompt))
          ]
  
      # visual dialog
      if "history" in samples.keys() and samples["history"][0] != "":
          prompt = [
              f'dialog history: {samples["history"][i]}\n{prompt[i]}'
              for i in range(len(prompt))
          ]
  
      if "caption" in samples.keys() and samples["caption"][0] != "":
          prompt = [
              f'This image has the caption "{samples["caption"][i]}". {prompt[i]}'
              for i in range(len(prompt))
          ]
  
      if (
          "discrn" in samples and self.use_caption
      ):  ## discriminatory reasoning
          if self.postfix:
              prompt = [f"{p}{self.postfix}" for p in prompt]
          if self.enumerate_inputs:
              prompt = [
                  f'{self.prefix}(a){Blip2VicunaXInstruct.MODALITY_TO_CUE[samples["modalities"][i][0]] if self.use_cues else " "}{samples["baseline_captions"][i][0]} (b){Blip2VicunaXInstruct.MODALITY_TO_CUE[samples["modalities"][i][1]] if self.use_cues else " "}{samples["baseline_captions"][i][1]} {prompt[i]}'
                  for i in range(bs)
              ]
          else:
              prompt = [
                  f'{self.prefix}{Blip2VicunaXInstruct.MODALITY_TO_CUE[samples["modalities"][i][0]]}{samples["baseline_captions"][i][0] if self.use_cues else " "}{Blip2VicunaXInstruct.MODALITY_TO_CUE[samples["modalities"][i][1]] if self.use_cues else " "}{samples["baseline_captions"][i][1]} {prompt[i]}'
                  for i in range(bs)
              ]
          text_input_tokens = self.llm_tokenizer(
              prompt, padding="longest", return_tensors="pt"
          ).to(self.device)
      else:
          if not self.projection_only and not getattr(
              self, f"projection_only_{modality}"
          ):
              query_tokens = {}
              for modality in self.modalities:
                  if modality not in samples:
                      continue
                  query_tokens[modality] = getattr(
                      self, f"{modality}_query_tokens"
                  ).expand(bs, -1, -1)
  
          if self.qformer_text_input:
              if (
                  self.special_qformer_input_prompt
                  or special_qformer_input_prompt
              ):
                  qformer_prompt = (
                      special_qformer_input_prompt
                      if special_qformer_input_prompt
                      else self.special_qformer_input_prompt
                  )
                  qformer_prompt = [qformer_prompt] * len(prompt)
                  if "text_input" in samples.keys():
                      if type(samples["text_input"][0]) == list:
                          qformer_prompt = [
                              qformer_prompt[i].format(
                                  *samples["text_input"][i]
                              )
                              for i in range(len(qformer_prompt))
                          ]
                      else:
                          qformer_prompt = [
                              qformer_prompt[i].format(
                                  samples["text_input"][i]
                              )
                              for i in range(len(qformer_prompt))
                          ]
  
                  text_Qformer = self.tokenizer(
                      qformer_prompt,
                      padding="longest",
                      truncation=True,
                      max_length=self.max_txt_len,
                      return_tensors="pt",
                  ).to(self.device)
              elif self.use_describe:
                  modality2prompt = {
                      "video": "a short description of the video",
                      "audio": "an audio that shows",
                      "image": "a short image caption",
                      "pc": "a 3d model of",
                  }
                  qformer_prompt = [
                      modality2prompt[modality]
                      for _ in samples["text_input"]
                  ]
  
                  # qformer_prompt = [f'Describe the {Blip2VicunaXInstruct.MODALITY_TO_CUE[modality].replace(":", "").strip() if modality != "pc" else "3d model"}.' for _ in samples["text_input"]]
                  text_Qformer = self.tokenizer(
                      qformer_prompt,
                      padding="longest",
                      truncation=True,
                      max_length=self.max_txt_len,
                      return_tensors="pt",
                  ).to(self.device)
              else:
                  text_Qformer = self.tokenizer(
                      prompt,
                      padding="longest",
                      truncation=True,
                      max_length=self.max_txt_len,
                      return_tensors="pt",
                  ).to(self.device)
  
              Qformer_atts = {}
              query_atts = {}
  
              for modality in curr_modalities:
                  # B, Token Size
                  query_atts[modality] = torch.ones(
                      query_tokens[modality].size()[:-1],
                      dtype=torch.long,
                  ).to(self.device)
                  # B, Token Size + Inp Size
                  Qformer_atts[modality] = torch.cat(
                      [
                          query_atts[modality],
                          text_Qformer.attention_mask,
                      ],
                      dim=1,
                  )
  
          embeds = {}
          data_atts = {}
          for modality in curr_modalities:
              data = samples[modality]
              ln = getattr(self, f"{modality}_ln")
              encoder = getattr(self, f"{modality}_encoder")
              if modality == "video" and "clip" in self.video_enc_name:
                  embeds[modality] = []
                  data_atts[modality] = []
                  for j in range(data.size(2)):
                      this_frame = data[:, :, j, :, :]
                      with self.maybe_autocast():
                          embeds[modality].append(
                              ln(encoder(this_frame))
                          )
                          if self.shared_qformer:
                              embeds[modality][j] = getattr(
                                  self, f"{modality}_encoder_projection"
                              )(embeds[modality][j])
                          data_atts[modality].append(
                              torch.ones(
                                  embeds[modality][j].size()[:-1],
                                  dtype=torch.long,
                              ).to(self.device)
                          )
  
              elif (
                  modality == "audio" and "beats" in self.audio_enc_name
              ):
                  embeds[modality] = []
                  data_atts[modality] = []
                  for j in range(data.size(1)):
                      this_frame = data[:, j, :, :]
                      with self.maybe_autocast():
                          embeds[modality].append(
                              ln(encoder(this_frame))
                          )
                          if self.shared_qformer:
                              embeds[modality][j] = getattr(
                                  self, f"{modality}_encoder_projection"
                              )(embeds[modality][j])
                      data_atts[modality].append(
                          torch.ones(
                              embeds[modality][j].size()[:-1],
                              dtype=torch.long,
                          ).to(self.device)
                      )
              else:
                  with self.maybe_autocast():
                      embeds[modality] = ln(encoder(data))
                  if len(embeds[modality].size()) == 2:
                      # B, C, D
                      embeds[modality] = embeds[modality].unsqueeze(1)
                  # B, C
                  if self.shared_qformer:
                      embeds[modality] = getattr(
                          self, f"{modality}_encoder_projection"
                      )(embeds[modality])
                  data_atts[modality] = torch.ones(
                      embeds[modality].size()[:-1], dtype=torch.long
                  ).to(self.device)
  
          query_outputs = {}
          num = {}
          if self.qformer_text_input:
              for modality in curr_modalities:
                  if (
                      modality
                      in Blip2VicunaXInstruct.SEQUENCIAL_MODALITIES
                      and getattr(self, f"{modality}_enc_name")
                      in Blip2VicunaXInstruct.SEQUENCIAL_ENCODERS
                  ):
                      num[modality] = len(embeds[modality])
                      bs = embeds[modality][0].shape[0]
                      indices = [
                          j_ + r
                          for r, j in enumerate(
                              [[i * bs for i in range(num[modality])]]
                              * bs
                          )
                          for j_ in j
                      ]
                      reordered_embeds = torch.cat(embeds[modality])[
                          indices
                      ]
                      reordered_atts = torch.cat(data_atts[modality])[
                          indices
                      ]
                      if self.projection_only or getattr(
                          self, f"projection_only_{modality}"
                      ):
                          if self.proj_dim != 1:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  reordered_embeds.mean(1, keepdim=True)
                              ).view(
                                  bs * num[modality],
                                  self.num_query_token,
                                  -1,
                              )
                          else:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  reordered_embeds.view(
                                      reordered_embeds.shape[0], -1
                                  )
                              )
                          continue
                      query_output = getattr(
                          self, f"{modality}_Qformer"
                      ).bert(
                          text_Qformer.input_ids.repeat(
                              num[modality], 1
                          ),
                          attention_mask=Qformer_atts[modality].repeat(
                              num[modality], 1
                          ),
                          query_embeds=query_tokens[modality].repeat(
                              num[modality], 1, 1
                          ),
                          encoder_hidden_states=reordered_embeds,
                          encoder_attention_mask=reordered_atts,
                          return_dict=True,
                      )
                      query_outputs[modality] = query_output
                  else:
                      bs = embeds[modality].shape[0]
                      if self.projection_only or getattr(
                          self, f"projection_only_{modality}"
                      ):
                          if self.proj_dim != 1:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  embeds[modality].mean(1, keepdim=True)
                              ).reshape(
                                  bs, self.num_query_token, -1
                              )
                          else:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(embeds[modality]).reshape(
                                  bs, self.num_query_token, -1
                              )
                          continue
                      query_outputs[modality] = getattr(
                          self, f"{modality}_Qformer"
                      ).bert(
                          text_Qformer.input_ids,
                          attention_mask=Qformer_atts[modality],
                          query_embeds=query_tokens[modality],
                          encoder_hidden_states=embeds[modality].to(
                              torch.float32
                          ),
                          encoder_attention_mask=data_atts[modality],
                          return_dict=True,
                      )
          else:
              for modality in curr_modalities:
                  if (
                      modality
                      in Blip2VicunaXInstruct.SEQUENCIAL_MODALITIES
                      and getattr(self, f"{modality}_enc_name")
                      in Blip2VicunaXInstruct.SEQUENCIAL_ENCODERS
                  ):
                      num[modality] = len(embeds[modality])
                      bs = embeds[modality][0].shape[0]
                      indices = [
                          j_ + r
                          for r, j in enumerate(
                              [[i * bs for i in range(num[modality])]]
                              * bs
                          )
                          for j_ in j
                      ]
                      reordered_embeds = torch.cat(embeds[modality])[
                          indices
                      ]
                      reordered_atts = torch.cat(data_atts[modality])[
                          indices
                      ]
                      if self.projection_only or getattr(
                          self, f"projection_only_{modality}"
                      ):
                          if self.proj_dim != 1:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  reordered_embeds.mean(1, keepdim=True)
                              ).view(
                                  bs * num[modality],
                                  self.num_query_token,
                                  -1,
                              )
                          else:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  reordered_embeds.view(
                                      reordered_embeds.shape[0], -1
                                  )
                              )
                          continue
                      query_output = getattr(
                          self, f"{modality}_Qformer"
                      ).bert(
                          query_embeds=query_tokens[modality].repeat(
                              num[modality], 1, 1
                          ),
                          encoder_hidden_states=reordered_embeds,
                          encoder_attention_mask=reordered_atts,
                          return_dict=True,
                      )
                      query_outputs[modality] = query_output
                  else:
                      bs = embeds[modality].shape[0]
                      if self.projection_only or getattr(
                          self, f"projection_only_{modality}"
                      ):
                          if self.proj_dim != 1:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(
                                  embeds[modality].mean(1, keepdim=True)
                              ).reshape(
                                  bs, self.num_query_token, -1
                              )
                          else:
                              query_outputs[modality] = getattr(
                                  self, f"{modality}_projection"
                              )(embeds[modality]).reshape(
                                  bs, self.num_query_token, -1
                              )
                          continue
                      query_outputs[modality] = getattr(
                          self, f"{modality}_Qformer"
                      ).bert(
                          query_embeds=query_tokens[modality],
                          encoder_hidden_states=embeds[modality].to(
                              torch.float32
                          ),
                          encoder_attention_mask=data_atts[modality],
                          return_dict=True,
                      )
  
          inputs_llm = {}
          atts_llm = {}
          enumeration = {}
          # from pdb import set_trace; set_trace()
          for i, modality in enumerate(curr_modalities):
              if (
                  modality in Blip2VicunaXInstruct.SEQUENCIAL_MODALITIES
                  and getattr(self, f"{modality}_enc_name")
                  in Blip2VicunaXInstruct.SEQUENCIAL_ENCODERS
              ):
                  if self.projection_only or getattr(
                      self, f"projection_only_{modality}"
                  ):
                      if self.proj_dim != 1:
                          inputs_llm[modality] = getattr(
                              self, f"{modality}_llm_proj"
                          )(
                              query_outputs[modality].unsqueeze(1)
                          ).reshape(
                              bs * num[modality],
                              self.num_query_token,
                              -1,
                          )
                      else:
                          inputs_llm[modality] = getattr(
                              self, f"{modality}_llm_proj"
                          )(
                              query_outputs[modality].reshape(
                                  bs * num, self.num_query_token, -1
                              )
                          )
                      inputs_llm[modality] = (
                          inputs_llm[modality]
                          .reshape(
                              bs,
                              num[modality],
                              self.num_query_token,
                              -1,
                          )
                          .view(
                              bs,
                              num[modality] * self.num_query_token,
                              -1,
                          )
                      )
                      atts_llm[modality] = torch.ones(
                          inputs_llm[modality].size()[:-1],
                          dtype=torch.long,
                      ).to(self.device)
                      continue
                  # num*bs, num query tokens, llm emb size
                  inputs_llm[modality] = getattr(
                      self, f"{modality}_llm_proj"
                  )(
                      query_outputs[modality].last_hidden_state[
                          :, : query_tokens[modality].size(1), :
                      ]
                  )
                  # bs, num, num query tokens, llm emb size -> bs, num*num query tokens, llm emb size
                  inputs_llm[modality] = (
                      inputs_llm[modality]
                      .reshape(
                          bs, num[modality], self.num_query_token, -1
                      )
                      .view(
                          bs, num[modality] * self.num_query_token, -1
                      )
                  )
                  atts_llm[modality] = torch.ones(
                      inputs_llm[modality].size()[:-1], dtype=torch.long
                  ).to(self.device)
  
              else:
                  if self.projection_only or getattr(
                      self, f"projection_only_{modality}"
                  ):
                      if self.proj_dim == 1:
                          inputs_llm[modality] = getattr(
                              self, f"{modality}_llm_proj"
                          )(query_outputs[modality].mean(-1)).reshape(
                              bs, self.num_query_token, -1
                          )
                      else:
                          inputs_llm[modality] = getattr(
                              self, f"{modality}_llm_proj"
                          )(
                              query_outputs[modality].reshape(
                                  bs, self.num_query_token, -1
                              )
                          )
                      atts_llm[modality] = torch.ones(
                          inputs_llm[modality].size()[:-1],
                          dtype=torch.long,
                      ).to(self.device)
                      continue
                  inputs_llm[modality] = getattr(
                      self, f"{modality}_llm_proj"
                  )(
                      query_outputs[modality]["last_hidden_state"][
                          :, : query_tokens[modality].size(1), :
                      ]
                  )
                  atts_llm[modality] = torch.ones(
                      inputs_llm[modality].size()[:-1], dtype=torch.long
                  ).to(self.device)
              if self.enumerate_inputs:
                  enumeration[modality] = self.llm_tokenizer(
                      [
                          f"{'' if i == 0 else ' '}({chr(97+i)}) "
                          for _ in prompt
                      ],
                      return_tensors="pt",
                      add_special_tokens=False
                      if (i != 0 or self.prefix)
                      else True,
                  ).to(self.device)
  
          att_list = []
          inp_list = []
          if self.prefix:
              att_list = [
                  self.tokenized_prefix.attention_mask.repeat(bs, 1).to(
                      self.device
                  )
              ]
              inp_list = [
                  self.llm_model.get_input_embeddings()(
                      self.tokenized_prefix.input_ids.to(self.device)
                  ).repeat(bs, 1, 1)
              ]
  
          for modality in curr_modalities:
              if self.enumerate_inputs:
                  enumeration_inputs_llm = (
                      self.llm_model.get_input_embeddings()(
                          enumeration[modality].input_ids.to(
                              self.device
                          )
                      )
                  )
                  enumeration_atts_llm = enumeration[
                      modality
                  ].attention_mask.to(self.device)
                  inp_list.extend([enumeration_inputs_llm])
                  att_list.extend([enumeration_atts_llm])
              if self.use_cues:
                  if self.clean_tokenization or self.remove_start:
                      if modality == curr_modalities[0] and not (
                          self.prefix or self.enumerate_inputs
                      ):
                          att_list.extend(
                              [
                                  torch.tensor(
                                      self.tokenized_cue[
                                          modality
                                      ].attention_mask
                                  )
                                  .to(self.device)
                                  .repeat(
                                      atts_llm[modality].shape[0], 1
                                  ),
                                  atts_llm[modality],
                              ]
                          )
                          inp_list.extend(
                              [
                                  self.emb_cue[modality]
                                  .to(self.device)
                                  .repeat(
                                      inputs_llm[modality].shape[0],
                                      1,
                                      1,
                                  ),
                                  inputs_llm[modality],
                              ]
                          )
                      else:
                          att_list.extend(
                              [
                                  torch.tensor(
                                      self.tokenized_cue[
                                          modality
                                      ].attention_mask[:, 1:]
                                  )
                                  .to(self.device)
                                  .repeat(
                                      atts_llm[modality].shape[0], 1
                                  ),
                                  atts_llm[modality],
                              ]
                          )
                          inp_list.extend(
                              [
                                  self.emb_cue[modality][:, 1:]
                                  .to(self.device)
                                  .repeat(
                                      inputs_llm[modality].shape[0],
                                      1,
                                      1,
                                  ),
                                  inputs_llm[modality],
                              ]
                          )
                  else:
                      att_list.extend(
                          [
                              torch.tensor(
                                  self.tokenized_cue[
                                      modality
                                  ].attention_mask
                              )
                              .to(self.device)
                              .repeat(atts_llm[modality].shape[0], 1),
                              atts_llm[modality],
                          ]
                      )
                      inp_list.extend(
                          [
                              self.emb_cue[modality]
                              .to(self.device)
                              .repeat(
                                  inputs_llm[modality].shape[0], 1, 1
                              ),
                              inputs_llm[modality],
                          ]
                      )
  
              else:
                  att_list.extend([atts_llm[modality]])
                  inp_list.extend([inputs_llm[modality]])
  
              if self.add_space:
                  space_tok = self.llm_tokenizer(
                      [f" " for _ in prompt],
                      return_tensors="pt",
                      add_special_tokens=False,
                  )
                  space_inputs_llm = (
                      self.llm_model.get_input_embeddings()(
                          space_tok.input_ids.to(self.device)
                      )
                  )
                  space_atts_llm = space_tok.attention_mask.to(
                      self.device
                  )
                  inp_list.extend([space_inputs_llm])
                  att_list.extend([space_atts_llm])
  
          atts_llm = torch.cat(att_list, dim=1)
          empty_targets = (
              torch.ones(atts_llm.size(), dtype=torch.long)
              .to(self.device)
              .fill_(-100)
          )
          inputs_llm = torch.cat(inp_list, dim=1)
  
          self.llm_tokenizer.padding_side = "right"
          self.llm_tokenizer.truncation_side = "left"
  
          text_input_tokens = self.llm_tokenizer(
              [f"{p}{self.postfix}" for p in prompt]
              if self.postfix
              else prompt,
              padding="longest",
              return_tensors="pt",
              add_special_tokens=not self.clean_tokenization,
          ).to(self.device)
  
      self.llm_tokenizer.truncation_side = "right"
      n_cands = len(candidates)
      with self.maybe_autocast():
          all_losses = []
          for n in range(n_segments):
              seg_len = n_cands // n_segments
              if n == (n_segments - 1):
                  seg_len = n_cands - seg_len * (n_segments - 1)
  
              start_i = n * (n_cands // n_segments)
              end_i = start_i + seg_len
              this_output_tokens = self.llm_tokenizer(
                  candidates[start_i:end_i],
                  return_tensors="pt",
                  padding="longest",
                  # truncation=True,
                  # max_length=self.max_output_txt_len,
              ).to(self.device)
  
              this_input_tokens_ids = (
                  text_input_tokens.input_ids.repeat_interleave(
                      seg_len, dim=0
                  )
              )
              this_input_tokens_atts = (
                  text_input_tokens.attention_mask.repeat_interleave(
                      seg_len, dim=0
                  )
              )
  
              this_output_tokens_ids = (
                  this_output_tokens.input_ids.repeat(bs, 1)
              )
              this_output_tokens_atts = (
                  this_output_tokens.attention_mask.repeat(bs, 1)
              )
  
              (
                  this_llm_tokens,
                  this_input_targets_len,
              ) = self.concat_text_input_output(
                  this_input_tokens_ids,
                  this_input_tokens_atts,
                  this_output_tokens_ids,
                  this_output_tokens_atts,
              )
  
              this_llm_input_ids = this_llm_tokens["input_ids"]
              this_llm_atts = this_llm_tokens["attention_mask"]
  
              inputs_embeds = self.llm_model.get_input_embeddings()(
                  this_llm_input_ids
              )
  
              if self.use_caption:
                  inputs_embeds = torch.cat([inputs_embeds], dim=1)
                  attention_mask = torch.cat([this_llm_atts], dim=1)
              else:
                  inputs_embeds = torch.cat(
                      [
                          inputs_llm.repeat_interleave(seg_len, dim=0),
                          inputs_embeds,
                      ],
                      dim=1,
                  )
                  attention_mask = torch.cat(
                      [
                          atts_llm.repeat_interleave(seg_len, dim=0),
                          this_llm_atts,
                      ],
                      dim=1,
                  )
  
              this_targets = this_llm_input_ids.masked_fill(
                  this_llm_input_ids == self.llm_tokenizer.pad_token_id,
                  -100,
              )
  
              for i, l in enumerate(this_input_targets_len):
                  this_targets[i][:l] = -100
  
              if self.use_caption:
                  torch.cat([this_targets], dim=1)
              else:
                  this_targets = torch.cat(
                      [
                          empty_targets.repeat_interleave(
                              seg_len, dim=0
                          ),
                          this_targets,
                      ],
                      dim=1,
                  )
  
              outputs = self.llm_model(
                  inputs_embeds=inputs_embeds,
                  attention_mask=attention_mask,
                  return_dict=True,
                  labels=this_targets,
                  reduction="none",
              )
  
              loss = outputs.loss
  
              loss = loss.reshape(bs, seg_len)
              all_losses.append(loss)
  
      all_losses = torch.cat(all_losses, dim=-1)
      all_losses = -all_losses
      output_class_ranks = torch.argsort(all_losses, dim=-1)
      return {
          "predictions": all_losses,
          "targets": torch.tensor(
              [self.candidates.index(l) for l in samples["label"]]
          ),
      }
  
imports: import logging
  import string
  from packaging import version
  import os
  from omegaconf import OmegaConf
  import torch
  from torch.cuda.amp import autocast
  import torch.nn
  from torch.nn.modules.module import _IncompatibleKeys
  from peft import get_peft_model, LoraConfig, TaskType
  import transformers
  import random
  from lavis.common.registry import registry
  from lavis.models.base_model import BaseModel
  from lavis.models.blip2_models.blip2 import Blip2Base, disabled_train, LayerNorm
  from lavis.models.ulip_models.ULIP_models import ULIP_PointBERT
  from lavis.tasks.multimodal_classification import MultimodalClassificationTask
  from lavis.common.utils import is_url
  from lavis.models.blip2_models.Qformer import BertConfig, BertLMHeadModel
  from lavis.common.dist_utils import download_cached_file
  from lavis.processors.blip_processors import BlipCaptionProcessor
label_desc: Use image node_id 12 for calling the Blip2VicunaXInstruct obj's underlying member method code with example usage: obj._predict_class(samples, candidates, n_segments, special_qformer_input_prompt) and returns: dict, None