docstr: Args:
      labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
          Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
          config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
          (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
  
  Returns:
  
  Example:
  
  ```python
  >>> from PIL import Image
  >>> import requests
  >>> from transformers import AutoProcessor, LlavaForConditionalGeneration
  
  >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
  >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
  
  >>> prompt = "<image>\nUSER: What's the content of the image?\nASSISTANT:"
  >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
  >>> image = Image.open(requests.get(url, stream=True).raw)
  
  >>> inputs = processor(text=prompt, images=image, return_tensors="pt")
  
  >>> # Generate
  >>> generate_ids = model.generate(**inputs, max_length=30)
  >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
  "\nUSER: What's the content of the image?\nASSISTANT: The image features a stop sign on a street corner"
  ```
docstr_tok: ["Args",":","labels","(","`","torch.LongTensor","`","of","shape","`","(","batch_size",",","sequence_length",")","`",",","*","optional","*",")",":","Labels","for","computing","the","masked","language","modeling","loss",".","Indices","should","either","be","in","`","[","0",",","...",",","config.vocab_size","]","`","or","-100","(","see","`","input_ids","`","docstring",")",".","Tokens","with","indices","set","to","`","-100","`","are","ignored","(","masked",")",",","the","loss","is","only","computed","for","the","tokens","with","labels","in","`","[","0",",","...",",","config.vocab_size","]","`",".","Returns",":","Example",":","``","`","python",">",">",">","from","PIL","import","Image",">",">",">","import","requests",">",">",">","from","transformers","import","AutoProcessor",",","LlavaForConditionalGeneration",">",">",">","model","=","LlavaForConditionalGeneration.from_pretrained","(","``","llava-hf\/llava-1.5-7b-hf","''",")",">",">",">","processor","=","AutoProcessor.from_pretrained","(","``","llava-hf\/llava-1.5-7b-hf","''",")",">",">",">","prompt","=","``","<","image",">","\\nUSER",":","What","'s","the","content","of","the","image","?","\\nASSISTANT",":","''",">",">",">","url","=","``","https",":","\/\/www.ilankelman.org\/stopsigns\/australia.jpg","''",">",">",">","image","=","Image.open","(","requests.get","(","url",",","stream=True",")",".raw",")",">",">",">","inputs","=","processor","(","text=prompt",",","images=image",",","return_tensors=","''","pt","''",")",">",">",">","#","Generate",">",">",">","generate_ids","=","model.generate","(","*","*","inputs",",","max_length=30",")",">",">",">","processor.batch_decode","(","generate_ids",",","skip_special_tokens=True",",","clean_up_tokenization_spaces=False",")","[","0","]","''","\\nUSER",":","What","'s","the","content","of","the","image","?","\\nASSISTANT",":","The","image","features","a","stop","sign","on","a","street","corner","''","``","`"]
code: def forward(
      self,
      input_ids: torch.LongTensor = None,
      pixel_values: torch.FloatTensor = None,
      attention_mask: Optional[torch.Tensor] = None,
      position_ids: Optional[torch.LongTensor] = None,
      past_key_values: Optional[List[torch.FloatTensor]] = None,
      inputs_embeds: Optional[torch.FloatTensor] = None,
      vision_feature_layer: Optional[int] = None,
      vision_feature_select_strategy: Optional[str] = None,
      labels: Optional[torch.LongTensor] = None,
      use_cache: Optional[bool] = None,
      output_attentions: Optional[bool] = None,
      output_hidden_states: Optional[bool] = None,
      return_dict: Optional[bool] = None,
  ) -> Union[Tuple, LlavaCausalLMOutputWithPast]:
      r"""
      Args:
          labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
              Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
              config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
              (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
  
      Returns:
  
      Example:
  
      ```python
      >>> from PIL import Image
      >>> import requests
      >>> from transformers import AutoProcessor, LlavaForConditionalGeneration
  
      >>> model = LlavaForConditionalGeneration.from_pretrained("llava-hf/llava-1.5-7b-hf")
      >>> processor = AutoProcessor.from_pretrained("llava-hf/llava-1.5-7b-hf")
  
      >>> prompt = "<image>\nUSER: What's the content of the image?\nASSISTANT:"
      >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg"
      >>> image = Image.open(requests.get(url, stream=True).raw)
  
      >>> inputs = processor(text=prompt, images=image, return_tensors="pt")
  
      >>> # Generate
      >>> generate_ids = model.generate(**inputs, max_length=30)
      >>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
      "\nUSER: What's the content of the image?\nASSISTANT: The image features a stop sign on a street corner"
      ```"""
  
      output_attentions = (
          output_attentions
          if output_attentions is not None
          else self.config.output_attentions
      )
      output_hidden_states = (
          output_hidden_states
          if output_hidden_states is not None
          else self.config.output_hidden_states
      )
      return_dict = (
          return_dict
          if return_dict is not None
          else self.config.use_return_dict
      )
      vision_feature_layer = (
          vision_feature_layer
          if vision_feature_layer is not None
          else self.config.vision_feature_layer
      )
      vision_feature_select_strategy = (
          vision_feature_select_strategy
          if vision_feature_select_strategy is not None
          else self.config.vision_feature_select_strategy
      )
  
      if inputs_embeds is None:
          # 1. Extra the input embeddings
          inputs_embeds = self.get_input_embeddings()(input_ids)
  
          # 2. Merge text and images
          if pixel_values is not None and input_ids.shape[1] != 1:
              image_outputs = self.vision_tower(
                  pixel_values, output_hidden_states=True
              )
              # this is not memory efficient at all (output_hidden_states=True) will save all the hidden stated.
              selected_image_feature = image_outputs.hidden_states[
                  vision_feature_layer
              ]
  
              if vision_feature_select_strategy == "default":
                  selected_image_feature = selected_image_feature[:, 1:]
              elif vision_feature_select_strategy == "full":
                  selected_image_feature = selected_image_feature
              else:
                  raise ValueError(
                      f"Unexpected select feature strategy: {self.config.vision_feature_select_strategy}"
                  )
  
              image_features = self.multi_modal_projector(
                  selected_image_feature
              )
              (
                  inputs_embeds,
                  attention_mask,
                  labels,
                  position_ids,
              ) = self._merge_input_ids_with_image_features(
                  image_features,
                  inputs_embeds,
                  input_ids,
                  attention_mask,
                  labels,
              )
              if labels is None:
                  labels = torch.full_like(
                      attention_mask, self.config.ignore_index
                  ).to(torch.long)
          else:
              # In case input_ids.shape[1] == 1 & pixel_values==None & past_key_values != None, we are in the case of
              # generation with cache
              if (
                  past_key_values is not None
                  and pixel_values is not None
                  and input_ids.shape[1] == 1
              ):
                  # Retrieve the first layer to inspect the logits and mask out the hidden states
                  # that are set to 0
                  first_layer_past_key_value = past_key_values[0][0][
                      :, :, :, 0
                  ]
  
                  # Sum all dimensions of head_dim (-2) to avoid random errors such as: https://github.com/huggingface/transformers/pull/28032#issuecomment-1863691941
                  batch_index, non_attended_tokens = torch.where(
                      first_layer_past_key_value.float().sum(-2) == 0
                  )
  
                  # Get the target length
                  target_seqlen = (
                      first_layer_past_key_value.shape[-1] + 1
                  )
  
                  extended_attention_mask = torch.ones(
                      (
                          attention_mask.shape[0],
                          target_seqlen - attention_mask.shape[1],
                      ),
                      dtype=attention_mask.dtype,
                      device=attention_mask.device,
                  )
  
                  # Filter out only the tokens that can be un-attended, this can happen
                  # if one uses Llava + Fused modules where the cache on the
                  # first iteration is already big enough, or if one passes custom cache
                  valid_indices = (
                      non_attended_tokens
                      < extended_attention_mask.size(-1)
                  )
                  new_batch_index = batch_index[valid_indices]
                  new_non_attended_tokens = non_attended_tokens[
                      valid_indices
                  ]
  
                  # Zero-out the places where we don't need to attend
                  extended_attention_mask[
                      new_batch_index, new_non_attended_tokens
                  ] = 0
  
                  attention_mask = torch.cat(
                      (attention_mask, extended_attention_mask), dim=1
                  )
                  position_ids = (
                      torch.sum(attention_mask, dim=1).unsqueeze(-1) - 1
                  )
  
      outputs = self.language_model(
          attention_mask=attention_mask,
          position_ids=position_ids,
          past_key_values=past_key_values,
          inputs_embeds=inputs_embeds,
          use_cache=use_cache,
          output_attentions=output_attentions,
          output_hidden_states=output_hidden_states,
          return_dict=return_dict,
      )
  
      logits = outputs[0]
  
      loss = None
      if labels is not None:
          # Shift so that tokens < n predict n
          if attention_mask is not None:
              shift_attention_mask = attention_mask[..., 1:]
              shift_logits = logits[..., :-1, :][
                  shift_attention_mask.to(logits.device) != 0
              ].contiguous()
              shift_labels = labels[..., 1:][
                  shift_attention_mask.to(labels.device) != 0
              ].contiguous()
          else:
              shift_logits = logits[..., :-1, :].contiguous()
              shift_labels = labels[..., 1:].contiguous()
          # Flatten the tokens
          loss_fct = nn.CrossEntropyLoss()
          loss = loss_fct(
              shift_logits.view(-1, shift_logits.size(-1)),
              shift_labels.view(-1).to(shift_logits.device),
          )
  
      if not return_dict:
          output = (logits,) + outputs[1:]
          return (loss,) + output if loss is not None else output
  
      return LlavaCausalLMOutputWithPast(
          loss=loss,
          logits=logits,
          past_key_values=outputs.past_key_values,
          hidden_states=outputs.hidden_states,
          attentions=outputs.attentions,
      )
  
imports: from dataclasses import dataclass
  from typing import List, Optional, Tuple, Union
  import torch
  import torch.utils.checkpoint
  from torch import nn
  from ...None import PreTrainedModel
  from ...activations import ACT2FN
  from ...cache_utils import Cache
  from ...modeling_outputs import ModelOutput
  from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
  from ..auto import AutoModel, AutoModelForCausalLM
  from .configuration_llava import LlavaConfig
label_desc: Use image node_id 11 for calling the LlavaForConditionalGeneration obj's underlying member method code with example usage: obj.forward(input_ids, pixel_values, attention_mask, position_ids, past_key_values, inputs_embeds, vision_feature_layer, vision_feature_select_strategy, labels, use_cache, output_attentions, output_hidden_states, return_dict) and returns: LlavaCausalLMOutputWithPast, unknown