class_docstr: BLIP2 Vicuna model.
  Supported model types:
      - vicuna7b
      - vicuna13b
  Usage:
      >>> from lavis.models import load_model
      >>> model = load_model("blip2_vicuna_xinstruct", "vicuna7b")
class_docstr_tok: ["BLIP2","Vicuna","model",".","Supported","model","types",":","-","vicuna7b","-","vicuna13b","Usage",":",">",">",">","from","lavis.models","import","load_model",">",">",">","model","=","load_model","(","``","blip2_vicuna_xinstruct","''",",","``","vicuna7b","''",")"]
code: def __init__(
      self,
      modalities=["image", "pc", "audio", "video"],
      use_cues=True,
      num_query_token=32,
      qformer_text_input=True,
      llm_text_input=False,
      apply_lemmatizer=False,
      ## encoders
      image_model="eva_clip_g",
      pc_model="ulip2_pointbert",
      video_model="eva_clip_g",
      audio_model="beats",
      image_encoder_kwargs={
          "image_size": 224,
          "drop_path_rate": 0,
          "use_grad_checkpoint": False,
      },
      pc_encoder_kwargs={},
      video_encoder_kwargs={},
      audio_encoder_kwargs={},
      image_precision="fp16",
      pc_precision="fp16",
      video_precision="fp16",
      audio_precision="fp16",
      freeze_image=True,
      freeze_pc=True,
      freeze_video=True,
      freeze_audio=True,
      ## load pretrained parameters
      pretrained_image_qformer=None,
      pretrained_pc_qformer=None,
      pretrained_video_qformer=None,
      pretrained_audio_qformer=None,
      load_attention_image_qformer=False,
      load_attention_pc_qformer=False,
      load_attention_video_qformer=False,
      load_attention_audio_qformer=False,
      load_qformer_type_image="",
      load_qformer_type_pc="",
      load_qformer_type_video="",
      load_qformer_type_audio="",
      load_ln_type_image="",
      load_ln_type_pc="",
      load_ln_type_video="",
      load_ln_type_audio="",
      load_projection_image=True,
      load_projection_pc=True,
      load_projection_video=True,
      load_projection_audio=True,
      load_projection_type_image="",
      load_projection_type_pc="",
      load_projection_type_video="",
      load_projection_type_audio="",
      ## llm model parameters
      llm_model="",
      lora_model="",
      lora=False,
      ## generation parameters
      prompt="",
      prefix="",
      postfix="",
      max_txt_len=128,
      max_output_txt_len=256,
      special_qformer_input_prompt=False,
      enumerate_inputs=False,
      add_space=False,
      remove_start=False,
      clean_tokenization=False,  # if set to true removes whitespace from cue, and start token from prompt.
      ## shared Q-former setup
      shared_qformer=False,
      pretrained_shared_qformer=None,
      load_attention_shared_qformer=False,
      load_qformer_type_shared="",
      load_projection_shared=False,
      load_projection_type_shared="",
      encoder_projection_type_image="",
      encoder_projection_type_pc="",
      encoder_projection_type_video="",
      encoder_projection_type_audio="",
      shared_qformer_num_features=512,
      ## use cached features
      cached_audio=False,
      cached_image=False,
      cached_pc=False,
      cached_video=False,
      ## num features for modality (only needed in cached cases.)
      num_features_audio=768,
      num_features_image=1408,
      num_features_video=1408,
      num_features_pc=512,
      joint_video_audio=False,
      ## DisCRN
      use_caption=False,
      use_describe=False,
      ## classification setup
      predict_with_gen=False,
      format_candidates_prompt="{}",
      ## projection only parameters
      projection_only=False,
      projection_only_audio=False,
      projection_only_pc=False,
      projection_only_video=False,
      projection_only_image=False,
      projection_path_audio=False,
      projection_path_pc=False,
      projection_path_video=False,
      projection_path_image=False,
      proj_dim=1,
  ):
      super().__init__()
  
      transformers_version = version.parse(transformers.__version__)
      assert transformers_version >= version.parse(
          "4.28"
      ), "BLIP-2 Vicuna requires transformers>=4.28"
      from transformers import LlamaTokenizer
      from lavis.models.blip2_models.modeling_llama import (
          LlamaForCausalLM,
      )
  
      logging.info(f"Using modalities {modalities}")
      self.modalities = modalities
  
      logging.info(f"Shared Qformer is set to {shared_qformer}")
      self.shared_qformer = shared_qformer
  
      logging.info(
          f"Video-audio interleaving is set to {joint_video_audio}"
      )
      self.joint_video_audio = joint_video_audio
  
      logging.info(
          f"Using Spacy en_core_wb_sm lemmatizer is set to {apply_lemmatizer}"
      )
      self._lemmatizer = None
      self.apply_lemmatizer = apply_lemmatizer
  
      logging.info(
          f"Qformer text input {qformer_text_input} and LLM Text Input {llm_text_input}"
      )
      self.qformer_text_input = qformer_text_input
      self.llm_text_input = llm_text_input
  
      self.projection_only = projection_only
      self.proj_dim = proj_dim
      logging.info(
          f"Projection only setup is set to {projection_only} with dimension {proj_dim}"
      )
  
      for modality in self.modalities:
          setattr(
              self, f"cached_{modality}", locals()[f"cached_{modality}"]
          )
          if locals()[f"cached_{modality}"]:
              setattr(
                  self,
                  f"num_features_{modality}",
                  locals()[f"num_features_{modality}"],
              )
              logging.info(
                  f"Using cached {modality} representation with {getattr(self, f'num_features_{modality}')} embedding dim."
              )
  
      ### Initialize modality enoders ###
      for modality in self.modalities:
          modality_model = locals()[f"{modality}_model"]
          modality_precision = locals()[f"{modality}_precision"]
          modality_kwargs = locals()[f"{modality}_encoder_kwargs"]
          modality_kwargs["load_ln_path"] = (
              locals()[f"pretrained_shared_qformer"]
              if shared_qformer
              else locals()[f"pretrained_{modality}_qformer"]
          )
          setattr(
              self,
              f"projection_only_{modality}",
              locals()[f"projection_only_{modality}"],
          )
          setattr(
              self,
              f"projection_path_{modality}",
              locals()[f"projection_path_{modality}"],
          )
          modality_kwargs["load_ln_type"] = locals()[
              f"load_ln_type_{modality}"
          ]
          if (
              self.projection_only
              or locals()[f"projection_only_{modality}"]
          ):
              modality_kwargs["load_ln_path"] = getattr(
                  self, f"projection_path_{modality}"
              )
              modality_kwargs["load_ln_type"] = modality
          setattr(
              self,
              f"load_ln_type_{modality}",
              locals()[f"load_ln_type_{modality}"],
          )
          setattr(
              self,
              f"pretrained_{modality}_qformer",
              locals()[f"pretrained_{modality}_qformer"],
          )
          modality_encoder, modality_ln = getattr(
              self, f"init_{modality}_encoder"
          )(
              modality_model,
              precision=modality_precision,
              **modality_kwargs,
          )
  
          freeze_modality = locals()[f"freeze_{modality}"]
          cached_modality = locals()[f"cached_{modality}"]
          if cached_modality:
              setattr(self, f"{modality}_encoder", modality_encoder)
              setattr(self, f"{modality}_ln", modality_ln)
              continue
          if freeze_modality:
              for name, param in modality_encoder.named_parameters():
                  param.requires_grad = False
              modality_encoder = modality_encoder.eval()
              modality_encoder.train = disabled_train
              logging.info(f"freeze {modality} encoder")
  
          setattr(self, f"{modality}_encoder", modality_encoder)
          setattr(self, f"{modality}_ln", modality_ln)
  
      ##### Init QFormers ####
      self.tokenizer = self.init_tokenizer(
          truncation_side="left"
      )  # 30523 tokens.
      self.num_query_token = num_query_token
      if self.shared_qformer:
          logging.info(
              f"Initializing shared QFormer with {shared_qformer_num_features} \
              number of features and query tokens of length {num_query_token}"
          )
          setattr(
              self,
              f"pretrained_shared_qformer",
              pretrained_shared_qformer,
          )
          setattr(
              self,
              f"load_qformer_type_shared",
              load_qformer_type_shared,
          )
          (
              self.shared_Qformer,
              self.shared_query_tokens,
          ) = self.init_Qformer(
              num_query_token,
              shared_qformer_num_features,
              pretrained_qformer=pretrained_shared_qformer,
              load_attention=load_attention_shared_qformer,
              load_qformer_type=load_qformer_type_shared,
          )
  
          if not qformer_text_input:
              self.shared_Qformer.bert.embeddings.word_embeddings = None
              self.shared_Qformer.bert.embeddings.position_embeddings = (
                  None
              )
              for layer in self.shared_Qformer.bert.encoder.layer:
                  layer.output = None
                  layer.intermediate = None
          else:
              self.shared_Qformer.resize_token_embeddings(
                  len(self.tokenizer)
              )
          self.shared_Qformer.cls = None
  
          # Map shared Qformer by reference to all modalities.
          for modality in self.modalities:
              setattr(self, f"{modality}_Qformer", self.shared_Qformer)
              setattr(
                  self,
                  f"{modality}_query_tokens",
                  self.shared_query_tokens,
              )
              encoder_proj_type = locals()[
                  f"encoder_projection_type_{modality}"
              ]
              setattr(
                  self,
                  f"encoder_projection_type_{modality}",
                  locals()[f"encoder_projection_type_{modality}"],
              )
              modality_encoder_features = getattr(
                  self, f"{modality}_encoder"
              ).num_features
              setattr(
                  self,
                  f"{modality}_encoder_projection",
                  self.init_encoder_projection(
                      modality_encoder_features,
                      shared_qformer_num_features,
                      pretrained_shared_qformer,
                      encoder_proj_type,
                  ),
              )
      else:
          for modality in self.modalities:
              if getattr(self, f"cached_{modality}"):
                  modality_num_features = locals()[
                      f"num_features_{modality}"
                  ]
              else:
                  modality_num_features = getattr(
                      self, f"{modality}_encoder"
                  ).num_features
  
              setattr(
                  self,
                  f"pretrained_{modality}_qformer",
                  locals()[f"pretrained_{modality}_qformer"],
              )
              setattr(
                  self,
                  f"load_qformer_type_{modality}",
                  locals()[f"load_qformer_type_{modality}"],
              )
  
              setattr(
                  self,
                  f"projection_only_{modality}",
                  locals()[f"projection_only_{modality}"],
              )
              setattr(
                  self,
                  f"projection_path_{modality}",
                  locals()[f"projection_path_{modality}"],
              )
  
              if (
                  self.projection_only
                  or locals()[f"projection_only_{modality}"]
              ):
                  logging.info(f"Initializing {modality} projection")
                  setattr(self, f"pretrained_{modality}_qformer", False)
                  if modality == "audio" and proj_dim == 1:
                      modality_num_features *= 256  # hack to get full beats embedding. define better.
                  modality_projection = self.init_vicuna_projection(
                      modality_num_features,
                      num_query_token * proj_dim,
                      load_projection_path=getattr(
                          self, f"projection_path_{modality}"
                      ),
                      load_projection_type=modality,
                      projection_key=f"{modality}_projection",
                  )
                  setattr(
                      self,
                      f"{modality}_projection",
                      modality_projection,
                  )
              else:
                  logging.info(
                      f"Initializing {modality} QFormer and query tokens of length {num_query_token}"
                  )
                  (
                      modality_qformer,
                      modality_query_tokens,
                  ) = self.init_Qformer(
                      num_query_token,
                      modality_num_features,
                      pretrained_qformer=locals()[
                          f"pretrained_{modality}_qformer"
                      ],
                      load_attention=locals()[
                          f"load_attention_{modality}_qformer"
                      ],
                      load_qformer_type=locals()[
                          f"load_qformer_type_{modality}"
                      ],
                  )
  
                  if not qformer_text_input:
                      modality_qformer.bert.embeddings.word_embeddings = (
                          None
                      )
                      modality_qformer.bert.embeddings.position_embeddings = (
                          None
                      )
                      for layer in modality_qformer.bert.encoder.layer:
                          layer.output = None
                          layer.intermediate = None
                  else:
                      modality_qformer.resize_token_embeddings(
                          len(self.tokenizer)
                      )
                  modality_qformer.cls = None
                  setattr(self, f"{modality}_Qformer", modality_qformer)
                  setattr(
                      self,
                      f"{modality}_query_tokens",
                      modality_query_tokens,
                  )
  
      ### Set up LLM ###
      logging.info(f"Setting up llm model {llm_model}")
      self.lora = lora
      print(f"Lora is set to {self.lora}")
      self.llm_tokenizer = LlamaTokenizer.from_pretrained(
          llm_model, use_fast=False, truncation_side="left"
      )
      self.llm_tokenizer.add_special_tokens({"pad_token": "[PAD]"})
      self.llm_tokenizer.add_special_tokens({"bos_token": "</s>"})
      self.llm_tokenizer.add_special_tokens({"eos_token": "</s>"})
      self.llm_tokenizer.add_special_tokens({"unk_token": "</s>"})
      if self.lora:
          # https://github.com/lxe/llama-peft-tuner/blob/main/finetune_peft.py
          self.llm_model = LlamaForCausalLM.from_pretrained(
              llm_model, load_in_8bit=True, torch_dtype=torch.float16
          )
          self.llm_model.resize_token_embeddings(
              len(self.llm_tokenizer)
          )
          self.peft_config = LoraConfig(
              task_type=TaskType.CAUSAL_LM,
              r=8,
              lora_alpha=32,
              lora_dropout=0.1,
              target_modules=["q_proj", "v_proj"],
          )
          self.llm_model.gradient_checkpointing_enable()
          self.llm_model.enable_input_require_grads()
          self.llm_model.lm_head = CastOutputToFloat(
              self.llm_model.lm_head
          )
          self.llm_model.config.use_cache = False  # silence the warnings. Please re-enable for inference!
          self.llm_hidden_size = self.llm_model.config.hidden_size
          self.llm_model = get_peft_model(
              self.llm_model, self.peft_config
          )
          self.lora_model = lora_model
  
      else:
          self.llm_model = LlamaForCausalLM.from_pretrained(
              llm_model, torch_dtype=torch.float16
          )
          self.llm_model.resize_token_embeddings(
              len(self.llm_tokenizer)
          )
          self.llm_hidden_size = self.llm_model.config.hidden_size
  
          for name, param in self.llm_model.named_parameters():
              param.requires_grad = False
  
      # Load LM projections
      if self.shared_qformer and load_projection_shared:
          qformer = getattr(self, f"shared_Qformer")
          load_projection_path = locals()[f"load_projection_shared"]
          if load_projection_path:
              load_projection_path = locals()[
                  f"pretrained_shared_qformer"
              ]
          load_projection_type = locals()[
              f"load_projection_type_shared"
          ]
          setattr(self, f"load_projection_shared", load_projection_path)
          setattr(
              self,
              f"load_projection_type_shared",
              locals()[f"load_projection_type_shared"],
          )
          logging.info(f"Loading shared Qformer projection.")
          proj = self.init_vicuna_projection(
              qformer.config.hidden_size,
              self.llm_hidden_size,
              load_projection_path=load_projection_path,
          )
          # Map projection by reference to all modalities.
          for modality in self.modalities:
              setattr(self, f"{modality}_llm_proj", proj)
      else:
          for modality in self.modalities:
              load_projection_path = locals()[
                  f"load_projection_{modality}"
              ]
              if load_projection_path == True:
                  load_projection_path = locals()[
                      f"pretrained_{modality}_qformer"
                  ]
              load_projection_type = locals()[
                  f"load_projection_type_{modality}"
              ]
              setattr(
                  self,
                  f"load_projection_{modality}",
                  load_projection_path,
              )
              setattr(
                  self,
                  f"load_projection_type_{modality}",
                  load_projection_type,
              )
              if self.projection_only or getattr(
                  self, f"projection_only_{modality}"
              ):
                  proj = self.init_vicuna_projection(
                      self.num_query_token
                      if proj_dim == 1
                      else proj_dim,
                      self.num_query_token * self.llm_hidden_size
                      if proj_dim == 1
                      else self.llm_hidden_size,
                      load_projection_path=getattr(
                          self, f"projection_path_{modality}"
                      ),
                      load_projection_type=load_projection_type,
                  )
              else:
                  qformer = getattr(self, f"{modality}_Qformer")
                  proj = self.init_vicuna_projection(
                      qformer.config.hidden_size,
                      self.llm_hidden_size,
                      load_projection_path=load_projection_path,
                      load_projection_type=load_projection_type,
                  )
              setattr(self, f"{modality}_llm_proj", proj)
  
      self.clean_tokenization = clean_tokenization
      logging.info(
          f"Clean tokenization is set to {self.clean_tokenization}"
      )
  
      self.max_txt_len = max_txt_len
      self.max_output_txt_len = max_output_txt_len
      self.prompt = prompt
  
      self.prefix = prefix
      if self.prefix:
          self.tokenized_prefix = self.llm_tokenizer(
              self.prefix, return_tensors="pt"
          )
  
      self.postfix = postfix
      if type(self.postfix) != str or not self.postfix:
          self.postfix = ""
      logging.info(
          f"Using prefix set to {self.prefix} and postfix set to {self.postfix}."
      )
  
      self.use_cues = use_cues
      logging.info(f"Using cues set to {self.use_cues}.")
      if self.use_cues:
          logging.info(
              f"Modality to cue {Blip2VicunaXInstruct.MODALITY_TO_CUE}"
          )
          self.tokenized_cue = {}
          self.emb_cue = {}
          self.att_cue = {}
          for modality in self.modalities:
              if self.clean_tokenization:
                  Blip2VicunaXInstruct.MODALITY_TO_CUE[
                      modality
                  ] = Blip2VicunaXInstruct.MODALITY_TO_CUE[
                      modality
                  ].lstrip()
              self.tokenized_cue[modality] = self.llm_tokenizer(
                  Blip2VicunaXInstruct.MODALITY_TO_CUE[modality],
                  return_tensors="pt",
              )
              self.emb_cue[
                  modality
              ] = self.llm_model.get_input_embeddings()(
                  self.tokenized_cue[modality].input_ids.to(self.device)
              )
              self.att_cue[modality] = self.tokenized_cue[
                  modality
              ].attention_mask.to(self.device)
  
      ## generation parameters
      self.use_caption = use_caption
      self.use_describe = use_describe
      self.predict_with_gen = predict_with_gen
      self.format_candidates_prompt = format_candidates_prompt
      self.special_qformer_input_prompt = special_qformer_input_prompt
      self.enumerate_inputs = enumerate_inputs
      self.add_space = add_space
      self.remove_start = remove_start
      if self.projection_only:
          self.qformer_text_input = False
  
imports: import logging
  import string
  from packaging import version
  import os
  from omegaconf import OmegaConf
  import torch
  from torch.cuda.amp import autocast
  import torch.nn
  from torch.nn.modules.module import _IncompatibleKeys
  from peft import get_peft_model, LoraConfig, TaskType
  import transformers
  import random
  from lavis.common.registry import registry
  from lavis.models.base_model import BaseModel
  from lavis.models.blip2_models.blip2 import Blip2Base, disabled_train, LayerNorm
  from lavis.models.ulip_models.ULIP_models import ULIP_PointBERT
  from lavis.tasks.multimodal_classification import MultimodalClassificationTask
  from lavis.common.utils import is_url
  from lavis.models.blip2_models.Qformer import BertConfig, BertLMHeadModel
  from lavis.common.dist_utils import download_cached_file
  from lavis.processors.blip_processors import BlipCaptionProcessor
label_desc: Use image node_id 1 to create a new Blip2VicunaXInstruct object from inherited base classes: Blip2Base with example: obj = Blip2VicunaXInstruct(modalities, use_cues, num_query_token, qformer_text_input, llm_text_input, apply_lemmatizer, image_model, pc_model, video_model, audio_model, image_encoder_kwargs, pc_encoder_kwargs, video_encoder_kwargs, audio_encoder_kwargs, image_precision, pc_precision, video_precision, audio_precision, freeze_image, freeze_pc, freeze_video, freeze_audio, pretrained_image_qformer, pretrained_pc_qformer, pretrained_video_qformer, pretrained_audio_qformer, load_attention_image_qformer, load_attention_pc_qformer, load_attention_video_qformer, load_attention_audio_qformer, load_qformer_type_image, load_qformer_type_pc, load_qformer_type_video, load_qformer_type_audio, load_ln_type_image, load_ln_type_pc, load_ln_type_video, load_ln_type_audio, load_projection_image, load_projection_pc, load_projection_video, load_projection_audio, load_projection_type_image, load_projection_type_pc, load_projection_type_video, load_projection_type_audio, llm_model, lora_model, lora, prompt, prefix, postfix, max_txt_len, max_output_txt_len, special_qformer_input_prompt, enumerate_inputs, add_space, remove_start, clean_tokenization, shared_qformer, pretrained_shared_qformer, load_attention_shared_qformer, load_qformer_type_shared, load_projection_shared, load_projection_type_shared, encoder_projection_type_image, encoder_projection_type_pc, encoder_projection_type_video, encoder_projection_type_audio, shared_qformer_num_features, cached_audio, cached_image, cached_pc, cached_video, num_features_audio, num_features_image, num_features_video, num_features_pc, joint_video_audio, use_caption, use_describe, predict_with_gen, format_candidates_prompt, projection_only, projection_only_audio, projection_only_pc, projection_only_video, projection_only_image, projection_path_audio, projection_path_pc, projection_path_video, projection_path_image, proj_dim)