code: def __getitem__(self, idx):
      word_list, relation, head_d, tail_d, imgid = (
          self.data_dict["words"][idx],
          self.data_dict["relations"][idx],
          self.data_dict["heads"][idx],
          self.data_dict["tails"][idx],
          self.data_dict["imgids"][idx],
      )
      item_id = self.data_dict["dataid"][idx]
      # [CLS] ... <s> head </s> ... <o> tail <o/> .. [SEP]
      head_pos, tail_pos = head_d["pos"], tail_d["pos"]
      # insert <s> <s/> <o> <o/>
      extend_word_list = []
      for i in range(len(word_list)):
          if i == head_pos[0]:
              extend_word_list.append("<s>")
          if i == head_pos[1]:
              extend_word_list.append("</s>")
          if i == tail_pos[0]:
              extend_word_list.append("<o>")
          if i == tail_pos[1]:
              extend_word_list.append("</o>")
          extend_word_list.append(word_list[i])
      extend_word_list = " ".join(extend_word_list)  # list不会进行子词分词
      encode_dict = self.tokenizer.encode_plus(
          text=extend_word_list,
          max_length=self.max_seq,
          truncation=True,
          padding="max_length",
      )
      input_ids, token_type_ids, attention_mask = (
          encode_dict["input_ids"],
          encode_dict["token_type_ids"],
          encode_dict["attention_mask"],
      )
      input_ids, token_type_ids, attention_mask = (
          torch.tensor(input_ids),
          torch.tensor(token_type_ids),
          torch.tensor(attention_mask),
      )
  
      re_label = self.re_dict[relation]  # label to id
  
      # image process
      if self.img_path is not None:
          try:
              img_path = os.path.join(
                  os.path.join(self.args.cwd, self.img_path), imgid
              )
              img_path = img_path.replace("test", "train")
              image = Image.open(img_path).convert("RGB")
              image = self.processor.clip_processor(
                  images=image, return_tensors="pt"
              )["pixel_values"].squeeze()
          except:
              img_path = os.path.join(
                  os.path.join(self.args.cwd, self.img_path), "inf.png"
              )
              image = Image.open(img_path).convert("RGB")
              image = self.processor.clip_processor(
                  images=image, return_tensors="pt"
              )["pixel_values"].squeeze()
          if self.aux_img_path is not None:
              # 辅助图像
              aux_imgs = []
              aux_img_paths = []
              imgid = imgid.split(".")[0]
              if item_id in self.data_dict["aux_imgs"]:
                  aux_img_paths = self.data_dict["aux_imgs"][item_id]
                  # print(aux_img_paths)
                  aux_img_paths = [
                      os.path.join(
                          os.path.join(
                              self.args.cwd, self.aux_img_path
                          ),
                          path,
                      )
                      for path in aux_img_paths
                  ]
              # 大于3需要舍弃
              for i in range(min(3, len(aux_img_paths))):
                  aux_img = Image.open(aux_img_paths[i]).convert("RGB")
                  aux_img = self.processor.aux_processor(
                      images=aux_img, return_tensors="pt"
                  )["pixel_values"].squeeze()
                  aux_imgs.append(aux_img)
  
              # 小于3需要加padding-0
              for i in range(3 - len(aux_imgs)):
                  aux_imgs.append(
                      torch.zeros((3, self.aux_size, self.aux_size))
                  )
  
              aux_imgs = torch.stack(aux_imgs, dim=0)
              assert len(aux_imgs) == 3
  
              if self.rcnn_img_path is not None:
                  rcnn_imgs = []
                  rcnn_img_paths = []
                  if imgid in self.data_dict["rcnn_imgs"]:
                      rcnn_img_paths = self.data_dict["rcnn_imgs"][
                          imgid
                      ]
                      rcnn_img_paths = [
                          os.path.join(
                              os.path.join(
                                  self.args.cwd, self.rcnn_img_path
                              ),
                              path,
                          )
                          for path in rcnn_img_paths
                      ]
                  # 大于3需要舍弃
                  for i in range(min(3, len(rcnn_img_paths))):
                      rcnn_img = Image.open(rcnn_img_paths[i]).convert(
                          "RGB"
                      )
                      rcnn_img = self.processor.rcnn_processor(
                          images=rcnn_img, return_tensors="pt"
                      )["pixel_values"].squeeze()
                      rcnn_imgs.append(rcnn_img)
                  # 小于3需要加padding-0
                  for i in range(3 - len(rcnn_imgs)):
                      rcnn_imgs.append(
                          torch.zeros(
                              (3, self.rcnn_size, self.rcnn_size)
                          )
                      )
  
                  rcnn_imgs = torch.stack(rcnn_imgs, dim=0)
                  assert len(rcnn_imgs) == 3
                  return (
                      input_ids,
                      token_type_ids,
                      attention_mask,
                      torch.tensor(re_label),
                      image,
                      aux_imgs,
                      rcnn_imgs,
                  )
  
              return (
                  input_ids,
                  token_type_ids,
                  attention_mask,
                  torch.tensor(re_label),
                  image,
                  aux_imgs,
              )
  
      return (
          input_ids,
          token_type_ids,
          attention_mask,
          torch.tensor(re_label),
      )
  
file_path: DeepKE/src/deepke/relation_extraction/multimodal/modules/dataset.py
imports: import random
  import os
  import torch
  import json
  import ast
  from PIL import Image
  from torch.utils.data import Dataset
  from transformers import BertTokenizer, CLIPProcessor
  import logging
label_desc: Use image node_id 3 for calling the MMREDataset obj's underlying member method code with example usage: obj.__getitem__(idx) and returns: input_ids,  token_type_ids,  attention_mask,  torch, input_ids,  token_type_ids,  attention_mask,  torch,  image,  aux_imgs, input_ids,  token_type_ids,  attention_mask,  torch,  image,  aux_imgs,  rcnn_imgs