code: def prepare_inputs_labels_for_multimodal(
      self,
      input_ids,
      attention_mask,
      past_key_values,
      labels,
      images,
      region_masks,
  ):
      if region_masks is not None:
          region_flag = True
      else:
          region_flag = False
      region_geo_sampler = region_flag and getattr(
          self.config, "region_geo_sampler", False
      )
  
      vision_tower = self.get_vision_tower()
      if (
          vision_tower is None
          or images is None
          or input_ids.shape[1] == 1
      ):
          if (
              past_key_values is not None
              and vision_tower is not None
              and images is not None
              and input_ids.shape[1] == 1
          ):
              attention_mask = torch.ones(
                  (
                      attention_mask.shape[0],
                      past_key_values[-1][-1].shape[-2] + 1,
                  ),
                  dtype=attention_mask.dtype,
                  device=attention_mask.device,
              )
          return (
              input_ids,
              attention_mask,
              past_key_values,
              None,
              labels,
          )
  
      if type(images) is list or images.ndim == 5:
          assert region_flag == False
          concat_images = torch.cat([image for image in images], dim=0)
          (
              raw_image_features,
              image_features,
              region_feature_map,
          ) = self.encode_images(
              concat_images, region_flag, region_geo_sampler
          )
          # image_features = self.encode_images(concat_images)
          split_sizes = [image.shape[0] for image in images]
          image_features = torch.split(
              image_features, split_sizes, dim=0
          )
          image_features = [x.flatten(0, 1) for x in image_features]
      else:
          (
              raw_image_features,
              image_features,
              region_feature_map,
          ) = self.encode_images(
              images, region_flag, region_geo_sampler
          )
  
      if region_flag:
          if region_geo_sampler:
              # pdb.set_trace()
              region_features = self.get_model().region_geo_sampler(
                  region_feature_map,
                  region_masks,
                  original_dtype=raw_image_features.dtype,
                  return_dtype=image_features.dtype,
              )
          else:
              region_features = self.extract_region_feature(
                  region_feature_map,
                  region_masks,
                  original_dtype=raw_image_features.dtype,
                  return_dtype=image_features.dtype,
              )
          assert len(region_features) == len(input_ids)
  
      new_input_embeds = []
      new_labels = [] if labels is not None else None
      cur_image_idx = 0
      for batch_idx, cur_input_ids in enumerate(input_ids):
          if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
              # multimodal LLM, but the current sample is not multimodal
              cur_input_embeds = self.get_model().embed_tokens(
                  cur_input_ids
              )
              cur_input_embeds = (
                  cur_input_embeds
                  + (
                      0.0
                      * self.get_model().mm_projector(
                          vision_tower.dummy_feature
                      )
                  ).sum()
              )
              new_input_embeds.append(cur_input_embeds)
              if labels is not None:
                  new_labels.append(labels[batch_idx])
              cur_image_idx += 1
              continue
          image_token_indices = torch.where(
              cur_input_ids == IMAGE_TOKEN_INDEX
          )[0]
          cur_new_input_embeds = []
          if labels is not None:
              cur_labels = labels[batch_idx]
              cur_new_labels = []
              assert cur_labels.shape == cur_input_ids.shape
          while image_token_indices.numel() > 0:
              cur_image_features = image_features[cur_image_idx]
              image_token_start = image_token_indices[0]
              if region_flag:
                  assert (
                      cur_input_ids[:image_token_start]
                      == self.config.im_region_fea_token
                  ).sum() == 0
              # If not use start-end token, pt ckpt saved only has mm projector.
              if getattr(
                  self.config, "tune_mm_mlp_adapter", False
              ) and getattr(self.config, "mm_use_im_start_end", False):
                  cur_new_input_embeds.append(
                      self.get_model()
                      .embed_tokens(
                          cur_input_ids[: image_token_start - 1]
                      )
                      .detach()
                  )
                  cur_new_input_embeds.append(
                      self.get_model().embed_tokens(
                          cur_input_ids[
                              image_token_start - 1 : image_token_start
                          ]
                      )
                  )
                  cur_new_input_embeds.append(cur_image_features)
                  cur_new_input_embeds.append(
                      self.get_model().embed_tokens(
                          cur_input_ids[
                              image_token_start
                              + 1 : image_token_start
                              + 2
                          ]
                      )
                  )
                  if labels is not None:
                      cur_new_labels.append(
                          cur_labels[:image_token_start]
                      )
                      cur_new_labels.append(
                          torch.full(
                              (cur_image_features.shape[0],),
                              IGNORE_INDEX,
                              device=labels.device,
                              dtype=labels.dtype,
                          )
                      )
                      cur_new_labels.append(
                          cur_labels[
                              image_token_start : image_token_start + 1
                          ]
                      )
                      cur_labels = cur_labels[image_token_start + 2 :]
              else:
                  cur_new_input_embeds.append(
                      self.get_model().embed_tokens(
                          cur_input_ids[:image_token_start]
                      )
                  )
                  cur_new_input_embeds.append(cur_image_features)
                  if labels is not None:
                      cur_new_labels.append(
                          cur_labels[:image_token_start]
                      )
                      cur_new_labels.append(
                          torch.full(
                              (cur_image_features.shape[0],),
                              IGNORE_INDEX,
                              device=labels.device,
                              dtype=labels.dtype,
                          )
                      )
                      cur_labels = cur_labels[image_token_start + 1 :]
              cur_image_idx += 1
              if getattr(
                  self.config, "tune_mm_mlp_adapter", False
              ) and getattr(self.config, "mm_use_im_start_end", False):
                  cur_input_ids = cur_input_ids[image_token_start + 2 :]
              else:
                  cur_input_ids = cur_input_ids[image_token_start + 1 :]
              image_token_indices = torch.where(
                  cur_input_ids == IMAGE_TOKEN_INDEX
              )[0]
          if cur_input_ids.numel() > 0:
              if getattr(
                  self.config, "tune_mm_mlp_adapter", False
              ) and getattr(self.config, "mm_use_im_start_end", False):
                  text_input_embeds = (
                      self.get_model()
                      .embed_tokens(cur_input_ids)
                      .detach()
                  )
              else:
                  text_input_embeds = self.get_model().embed_tokens(
                      cur_input_ids
                  )
              if labels is not None:
                  cur_new_labels.append(cur_labels)
  
              # Add region feature into text feature embeddings.
              assert batch_idx + 1 == cur_image_idx
              if region_flag and region_features[batch_idx] is not None:
                  region_embs = torch.zeros_like(text_input_embeds)
                  region_replace_mask = (
                      cur_input_ids == self.config.im_region_fea_token
                  )
                  # pdb.set_trace()
                  region_embs[region_replace_mask] = region_features[
                      batch_idx
                  ].to(text_input_embeds.dtype)
                  text_input_embeds = (
                      text_input_embeds
                      * (~region_replace_mask).to(
                          text_input_embeds.dtype
                      )[:, None]
                      + region_embs
                  )
                  # print('region_embs[..., 0].nonzero()', region_embs[..., 0].nonzero())
                  # raise NotImplementedError()
                  # pdb.set_trace()
              else:
                  if hasattr(self.config, "im_region_fea_token"):
                      assert (
                          cur_input_ids
                          == self.config.im_region_fea_token
                      ).sum() == 0
  
              cur_new_input_embeds.append(text_input_embeds)
          cur_new_input_embeds = [
              x.to(device=self.device) for x in cur_new_input_embeds
          ]
          cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0)
          new_input_embeds.append(cur_new_input_embeds)
          if labels is not None:
              cur_new_labels = torch.cat(cur_new_labels, dim=0)
              new_labels.append(cur_new_labels)
  
      if any(
          x.shape != new_input_embeds[0].shape for x in new_input_embeds
      ):
          max_len = max(x.shape[0] for x in new_input_embeds)
  
          new_input_embeds_align = []
          for cur_new_embed in new_input_embeds:
              cur_new_embed = torch.cat(
                  (
                      cur_new_embed,
                      torch.zeros(
                          (
                              max_len - cur_new_embed.shape[0],
                              cur_new_embed.shape[1],
                          ),
                          dtype=cur_new_embed.dtype,
                          device=cur_new_embed.device,
                      ),
                  ),
                  dim=0,
              )
              new_input_embeds_align.append(cur_new_embed)
          new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
  
          if labels is not None:
              new_labels_align = []
              _new_labels = new_labels
              for cur_new_label in new_labels:
                  cur_new_label = torch.cat(
                      (
                          cur_new_label,
                          torch.full(
                              (max_len - cur_new_label.shape[0],),
                              IGNORE_INDEX,
                              dtype=cur_new_label.dtype,
                              device=cur_new_label.device,
                          ),
                      ),
                      dim=0,
                  )
                  new_labels_align.append(cur_new_label)
              new_labels = torch.stack(new_labels_align, dim=0)
  
          if attention_mask is not None:
              new_attention_mask = []
              for (
                  cur_attention_mask,
                  cur_new_labels,
                  cur_new_labels_align,
              ) in zip(attention_mask, _new_labels, new_labels):
                  new_attn_mask_pad_left = torch.full(
                      (cur_new_labels.shape[0] - labels.shape[1],),
                      True,
                      dtype=attention_mask.dtype,
                      device=attention_mask.device,
                  )
                  new_attn_mask_pad_right = torch.full(
                      (
                          cur_new_labels_align.shape[0]
                          - cur_new_labels.shape[0],
                      ),
                      False,
                      dtype=attention_mask.dtype,
                      device=attention_mask.device,
                  )
                  cur_new_attention_mask = torch.cat(
                      (
                          new_attn_mask_pad_left,
                          cur_attention_mask,
                          new_attn_mask_pad_right,
                      ),
                      dim=0,
                  )
                  new_attention_mask.append(cur_new_attention_mask)
              attention_mask = torch.stack(new_attention_mask, dim=0)
              assert attention_mask.shape == new_labels.shape
      else:
          new_input_embeds = torch.stack(new_input_embeds, dim=0)
          if labels is not None:
              new_labels = torch.stack(new_labels, dim=0)
  
          if attention_mask is not None:
              new_attn_mask_pad_left = torch.full(
                  (
                      attention_mask.shape[0],
                      new_input_embeds.shape[1] - input_ids.shape[1],
                  ),
                  True,
                  dtype=attention_mask.dtype,
                  device=attention_mask.device,
              )
              attention_mask = torch.cat(
                  (new_attn_mask_pad_left, attention_mask), dim=1
              )
              assert attention_mask.shape == new_input_embeds.shape[:2]
  
      return (
          None,
          attention_mask,
          past_key_values,
          new_input_embeds,
          new_labels,
      )
  
imports: from abc import ABC, abstractmethod
  import torch
  import torch.nn
  import torch.nn.functional
  import math
  from .multimodal_encoder.builder import build_vision_tower
  import pdb
  from ferret.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
label_desc: Use image node_id 5 for calling the FERRETMetaForCausalLM obj's underlying member method code with example usage: obj.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images, region_masks) and returns: None,  attention_mask,  past_key_values,  new_input_embeds,  new_labels, input_ids,  attention_mask,  past_key_values,  None,  labels