class_docstr: BLIP2 OPT model.
  Supported model types:
      - pretrained_opt2.7b: pretrained model with OPT2.7b
      - pretrained_opt6.7b: pretrained model with OPT6.7b
      - caption_coco_opt2.7b: fintuned image captioning model with OPT2.7b
      - caption_coco_opt6.7b: fintuned image captioning model with OPT6.7b
  Usage:
      >>> from lavis.models import load_model
      >>> model = load_model("blip2_opt", "caption_coco_opt2.7b")
class_docstr_tok: ["BLIP2","OPT","model",".","Supported","model","types",":","-","pretrained_opt2.7b",":","pretrained","model","with","OPT2.7b","-","pretrained_opt6.7b",":","pretrained","model","with","OPT6.7b","-","caption_coco_opt2.7b",":","fintuned","image","captioning","model","with","OPT2.7b","-","caption_coco_opt6.7b",":","fintuned","image","captioning","model","with","OPT6.7b","Usage",":",">",">",">","from","lavis.models","import","load_model",">",">",">","model","=","load_model","(","``","blip2_opt","''",",","``","caption_coco_opt2.7b","''",")"]
docstr: apply_lemmatizer: when set to True, postprocess predict_answers() result with lemmas.
docstr_tok: ["apply_lemmatizer",":","when","set","to","True",",","postprocess","predict_answers","(",")","result","with","lemmas","."]
code: def __init__(
      self,
      vit_model="eva_clip_g",
      img_size=224,
      drop_path_rate=0,
      use_grad_checkpoint=False,
      vit_precision="fp16",
      freeze_vit=True,
      num_query_token=32,
      opt_model="facebook/opt-2.7b",
      prompt="",
      max_txt_len=32,
      apply_lemmatizer=False,
  ):
      """
      apply_lemmatizer: when set to True, postprocess predict_answers() result with lemmas.
      """
      super().__init__()
      transformers_version = version.parse(transformers.__version__)
      assert transformers_version >= version.parse(
          "4.27"
      ), "BLIP-2 OPT requires transformers>=4.27"
  
      self.tokenizer = self.init_tokenizer()
  
      self.visual_encoder, self.ln_vision = self.init_vision_encoder(
          vit_model,
          img_size,
          drop_path_rate,
          use_grad_checkpoint,
          vit_precision,
      )
      if freeze_vit:
          for name, param in self.visual_encoder.named_parameters():
              param.requires_grad = False
          self.visual_encoder = self.visual_encoder.eval()
          self.visual_encoder.train = disabled_train
          logging.info("freeze vision encoder")
  
      self.Qformer, self.query_tokens = self.init_Qformer(
          num_query_token, self.visual_encoder.num_features
      )
      self.Qformer.cls = None
      self.Qformer.bert.embeddings.word_embeddings = None
      self.Qformer.bert.embeddings.position_embeddings = None
      for layer in self.Qformer.bert.encoder.layer:
          layer.output = None
          layer.intermediate = None
  
      self.opt_tokenizer = AutoTokenizer.from_pretrained(
          opt_model, use_fast=False
      )
      self.opt_model = OPTForCausalLM.from_pretrained(
          opt_model, torch_dtype=torch.float16
      )
      for name, param in self.opt_model.named_parameters():
          param.requires_grad = False
      self.eos_token_id = self.opt_tokenizer(
          "\n", add_special_tokens=False
      ).input_ids[0]
  
      self.opt_proj = nn.Linear(
          self.Qformer.config.hidden_size,
          self.opt_model.config.hidden_size,
      )
  
      self.max_txt_len = max_txt_len
      self.prompt = prompt
      prompt_tokens = self.opt_tokenizer(
          self.prompt, return_tensors="pt"
      )
      self.prompt_length = prompt_tokens.attention_mask.sum(1)
  
      self._apply_lemmatizer = apply_lemmatizer
      self._lemmatizer = None
  
file_path: lavis/lavis/models/blip2_models/blip2_opt.py
imports: import logging
  from packaging import version
  import torch
  from torch.cuda.amp import autocast
  import torch.nn
  from lavis.common.registry import registry
  from lavis.models.blip2_models.blip2 import Blip2Base, disabled_train
  from transformers import AutoTokenizer, OPTForCausalLM, OPTConfig
  import transformers
label_desc: Use image node_id 1 to create a new Blip2OPT object from inherited base classes: Blip2Base with example: obj = Blip2OPT(vit_model, img_size, drop_path_rate, use_grad_checkpoint, vit_precision, freeze_vit, num_query_token, opt_model, prompt, max_txt_len, apply_lemmatizer)