class_docstr: BLIP2 Vicuna model.
  Supported model types:
      - vicuna7b
      - vicuna13b
  Usage:
      >>> from lavis.models import load_model
      >>> model = load_model("blip2_vicuna_xinstruct", "vicuna7b")
class_docstr_tok: ["BLIP2","Vicuna","model",".","Supported","model","types",":","-","vicuna7b","-","vicuna13b","Usage",":",">",">",">","from","lavis.models","import","load_model",">",">",">","model","=","load_model","(","``","blip2_vicuna_xinstruct","''",",","``","vicuna7b","''",")"]
code: def load_state_dict(self, state_dict, strict=True):
      # from pdb import set_trace; set_trace()
      unexpected_keys = []
      missing_keys = []
      if self.shared_qformer and not self.projection_only:
          ## Load Q-Former if it is not loaded from config
          if not getattr(self, "pretrained_shared_qformer"):
              shared_qformer_state_dict = {
                  ".".join(k.split(".")[1:]): v
                  for k, v in state_dict.items()
                  if "shared_Qformer" == k.split(".")[0]
              }
              msg = self.shared_Qformer.load_state_dict(
                  shared_qformer_state_dict, strict=strict
              )
              missing_keys.extend(msg.missing_keys)
              ## Load query tokens
              if "shared_query_tokens" not in state_dict:
                  missing_keys.append("shared_query_tokens")
              else:
                  self.shared_query_tokens = state_dict[
                      "shared_query_tokens"
                  ]
              missing_keys.extend(msg.missing_keys)
              unexpected_keys.extend(msg.unexpected_keys)
  
              for modality in self.modalities:
                  # Map shared Qformer by reference to all modalities.
                  setattr(
                      self, f"{modality}_Qformer", self.shared_Qformer
                  )
                  getattr(
                      self, f"{modality}_query_tokens"
                  ).data = state_dict[f"shared_query_tokens"]
                  # load encoder projections
                  modality_encoder_projection_dict = {
                      ".".join(k.split(".")[1:]): v
                      for k, v in state_dict.items()
                      if f"{modality}_encoder_projection"
                      in k.split(".")[0]
                  }
                  msg = getattr(
                      self, f"{modality}_encoder_projection"
                  ).load_state_dict(
                      modality_encoder_projection_dict, strict=strict
                  )
                  missing_keys.extend(msg.missing_keys)
                  unexpected_keys.extend(msg.unexpected_keys)
                  # load modality layer norm
                  if (
                      getattr(self, f"load_ln_type_{modality}")
                      == "vision"
                  ):
                      modality_ln_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"ln_vision" in k.split(".")[0]
                      }
                  else:
                      modality_ln_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"{modality}_ln" in k.split(".")[0]
                      }
                  msg = getattr(self, f"{modality}_ln").load_state_dict(
                      modality_ln_dict, strict=strict
                  )
                  missing_keys.extend(msg.missing_keys)
                  unexpected_keys.extend(msg.unexpected_keys)
  
          ## Load Shared LLM projection if not loaded by config
          if not getattr(self, "load_projection_shared"):
              shared_llm_projection_dict = {
                  ".".join(k.split(".")[1:]): v
                  for k, v in state_dict.items()
                  if f"shared_llm_proj" in k.split(".")[0]
              }
              msg = self.shared_llm_proj.load_state_dict(
                  shared_llm_projection_dict, strict=strict
              )
              missing_keys.extend(msg.missing_keys)
              unexpected_keys.extend(msg.unexpected_keys)
              for modality in self.modalities:
                  ## Map to modality projections by reference
                  msg = setattr(
                      self, f"{modality}_llm_proj", self.shared_llm_proj
                  )
      else:
          for modality in self.modalities:
              ## Load Q-Former if not loaded from config
              if not getattr(
                  self, f"pretrained_{modality}_qformer"
              ) or (
                  (
                      self.projection_only
                      or getattr(self, f"projection_only_{modality}")
                  )
                  and not getattr(self, f"projection_path_{modality}")
              ):
                  if self.projection_only or getattr(
                      self, f"projection_only_{modality}"
                  ):
                      if not getattr(
                          self, f"projection_path_{modality}"
                      ):
                          logging.info(f"Loaded {modality} projection")
                          modality_qformer_state_dict = {
                              ".".join(k.split(".")[1:]): v
                              for k, v in state_dict.items()
                              if f"{modality}_projection"
                              == k.split(".")[0]
                          }
                          msg = getattr(
                              self, f"{modality}_projection"
                          ).load_state_dict(
                              modality_qformer_state_dict, strict=strict
                          )
                          missing_keys.extend(msg.missing_keys)
                          unexpected_keys.extend(msg.unexpected_keys)
                  else:
                      modality_qformer_state_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"{modality}_Qformer" == k.split(".")[0]
                      }
                      msg = getattr(
                          self, f"{modality}_Qformer"
                      ).load_state_dict(
                          modality_qformer_state_dict, strict=strict
                      )
                      missing_keys.extend(msg.missing_keys)
                      unexpected_keys.extend(msg.unexpected_keys)
                  ## Load query tokens
                  if not self.projection_only and not getattr(
                      self, f"projection_only_{modality}"
                  ):
                      if f"{modality}_query_tokens" not in state_dict:
                          missing_keys.append(
                              f"{modality}_query_tokens"
                          )
                      else:
                          logging.info(
                              f"Loaded {modality} query tokens"
                          )
                          getattr(
                              self, f"{modality}_query_tokens"
                          ).data = state_dict[
                              f"{modality}_query_tokens"
                          ]
                  # load modality layer norm if not loaded from config
                  if (
                      getattr(self, f"load_ln_type_{modality}")
                      == "vision"
                  ):
                      logging.info(f"Loaded {modality} vision ln")
                      modality_ln_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"ln_vision" in k.split(".")[0]
                      }
                  else:
                      modality_ln_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"{modality}_ln" in k.split(".")[0]
                      }
                  msg = getattr(self, f"{modality}_ln").load_state_dict(
                      modality_ln_dict, strict=strict
                  )
                  missing_keys.extend(msg.missing_keys)
                  unexpected_keys.extend(msg.unexpected_keys)
              ## Load LLM projections if not loaded from config
              if not getattr(self, f"load_projection_{modality}") or (
                  getattr(self, f"projection_only_{modality}")
                  or self.projection_only
              ):
                  if not getattr(self, f"projection_path_{modality}"):
                      logging.info(f"Loaded {modality} llm  projection")
                      modality_llm_projection_dict = {
                          ".".join(k.split(".")[1:]): v
                          for k, v in state_dict.items()
                          if f"{modality}_llm_proj" in k.split(".")[0]
                      }
                      msg = getattr(
                          self, f"{modality}_llm_proj"
                      ).load_state_dict(
                          modality_llm_projection_dict, strict=strict
                      )
                      missing_keys.extend(msg.missing_keys)
                      unexpected_keys.extend(msg.unexpected_keys)
  
      ## llm model is loaded from pretrained
      lora_state_dict = {
          ".".join(k.split(".")[1:]): v
          for k, v in state_dict.items()
          if f"llm_model" in k.split(".")[0]
      }
  
      if not self.lora or len(lora_state_dict) == 0:
          unexpected_keys = [
              k
              for k in unexpected_keys
              if k.split(".")[0] != "llm_model"
          ]
      else:
          msg = self.llm_model.load_state_dict(
              {
                  ".".join(k.split(".")[1:]): v
                  for k, v in state_dict.items()
                  if f"llm_model" in k.split(".")[0]
              },
              strict=False,
          )
          missing_keys.extend(
              ["llm_model." + k for k in msg.missing_keys]
          )
      missing_keys = [
          k for k in missing_keys if "encoder" not in k.split(".")[0]
      ]
      missing_keys = [
          k for k in missing_keys if k.split(".")[0] != "llm_model"
      ]
      return _IncompatibleKeys(missing_keys, unexpected_keys)
  
imports: import logging
  import string
  from packaging import version
  import os
  from omegaconf import OmegaConf
  import torch
  from torch.cuda.amp import autocast
  import torch.nn
  from torch.nn.modules.module import _IncompatibleKeys
  from peft import get_peft_model, LoraConfig, TaskType
  import transformers
  import random
  from lavis.common.registry import registry
  from lavis.models.base_model import BaseModel
  from lavis.models.blip2_models.blip2 import Blip2Base, disabled_train, LayerNorm
  from lavis.models.ulip_models.ULIP_models import ULIP_PointBERT
  from lavis.tasks.multimodal_classification import MultimodalClassificationTask
  from lavis.common.utils import is_url
  from lavis.models.blip2_models.Qformer import BertConfig, BertLMHeadModel
  from lavis.common.dist_utils import download_cached_file
  from lavis.processors.blip_processors import BlipCaptionProcessor
label_desc: Use image node_id 24 for calling the Blip2VicunaXInstruct obj's underlying member method code with example usage: obj.load_state_dict(state_dict, strict) and returns: _IncompatibleKeys