IndexError: index 39092 is out of bounds for axis 0 with size 39092; I'm trying to train a multi label classification
I think the problem is caused by the way I load the data from the csv file, but I don't know how to fix it
Here is a small portion of my train csv file:
train_datasets
(I use 15 labels from column 1 to the end)
The error looks like this:
IndexError
IndexError
Code==&gt;
import csv
import os
import pandas as pd
import numpy as np
from PIL import Image
import torch

class FashionData(torch.utils.data.Dataset):
    def __init__(self, csv_file, mode='train', transform=None):
        self.mode=mode
        #label(img1) = [0, 0, 0, 1], label(img3) = [1, 0, 1, 0], ...),
        self.transform = transform


        self.data_info = pd.read_csv(csv_file, header=None)
        #print(self.data_info)
        
        
        # First column contains the image paths
        self.image_arr = np.asarray(self.data_info.iloc[1:, 0])
        if mode !='test':
            self.label_arr = np.asarray(self.data_info.iloc[1:, 2:]) # columns 1 to N
            self.label_arr=self.label_arr.astype('float32')

        
        # Calculate len
        self.data_len = len(self.data_info.index)
    def __getitem__(self, index):
        # Get image name from the pandas df
        single_image_name = self.image_arr[index]
        # Open image
        img_as_img = Image.open(single_image_name)
        
        if self.transform is not None:
            img_as_img = self.transform(img_as_img)
        if self.mode=='test':
            return img_as_img

        # Get label(class) of the image based on the cropped pandas column
        single_image_label = self.label_arr[index]
        #single_image_label = torch.from_numpy(self.label_arr[index]).float()
        #img = torch.from_numpy(img).float().to(device)
        #label = torch.tensor(int(self.labels[index]))

        return (img_as_img, single_image_label)

    def __len__(self):
        return self.data_len

transforms_train = transforms.Compose([transforms.Resize((224,224)),transforms.RandomHorizontalFlip(),transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
transforms_test = transforms.Compose([transforms.Resize((224,224)),transforms.ToTensor(),transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
dataset_train = FashionData('./deep_fashion/train.csv', mode='train', transform=transforms_train)
dataset_val = FashionData('./deep_fashion/val.csv', mode='val', transform=transforms_test)
dataset_test = FashionData('./deep_fashion/test.csv', mode='test', transform=transforms_test)

from torch.utils.data import DataLoader

train_loader = DataLoader(dataset_train, batch_size=128, shuffle=True)
val_loader = DataLoader(dataset_val, batch_size=128, shuffle=False)
test_loader = DataLoader(dataset_test, batch_size=128, shuffle=False)

model=models.resnet50(pretrained=True)
for params in model.parameters():
  params.requires_grad=False
model.fc=nn.Sequential(
    nn.Linear(2048,15),
    nn.Sigmoid()
    )

device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model=model.to(device)
print(model)

criterion=nn.BCELoss()
#criterion=nn.BCEWithLogitsLoss()
optimizer=optim.Adam(model.parameters(),lr=0.001)
criterion=criterion.to(device)

def train(train_loader,model,criterion,optimizer):
  model.train()
  loss_list=[]
  total_count=0
  acc_count=0
  
  for x,y in train_loader:
    
    x=x.to(device)
    y=y.to(device)

    optimizer.zero_grad()
    output=model(x)
    
    loss=criterion(output,y)
    loss.backward()
    optimizer.step()
    
    #_,predicted=torch.max(output,1)
    predicted=(output&gt;0.5).float()
    total_count+=y.size(0)
    acc_count+=(predicted==y).sum().item()
    
    loss_list.append(loss.item())
  acc=acc_count/total_count
  loss=sum(loss_list)/len(loss_list)
  return acc, loss

def val(valid_loader,model,criterion):
  model.eval()
  loss_list=[]
  total_count=0
  acc_count=0
  with torch.no_grad():
    for x,y in valid_loader:
      x=x.to(device)
      
      y=y.to(device)
      
      output=model(x)
      loss=criterion(output,y)
      #_,predicted=torch.max(output,1)
      predicted=(output&gt;0.5).float()
    
      total_count+=y.size(0)
      acc_count+=(predicted==y).sum().item()
      loss_list.append(loss.item())

  acc=acc_count/total_count
  loss=sum(loss_list)/len(loss_list)
  return acc, loss

train_acc_list = []
train_loss_list = []
val_acc_list = []
val_loss_list = []

for epoch in range(10):
    train_acc, train_loss = train(train_loader, model, criterion, optimizer)
    val_acc, val_loss=val(val_loader, model, criterion)
    train_acc_list.append(train_acc)
    train_loss_list.append(train_loss)
    val_acc_list.append(val_acc)
    val_loss_list.append(val_loss)
    print('epoch',epoch)     
    print('Train Acc: {:.6f} Train Loss: {:.6f}'.format(train_acc, train_loss))
    print('  Val Acc: {:.6f}   Val Loss: {:.6f}'.format(val_acc, val_loss))



Did I do wrong in the loading data part? Or it's the other problem?

11111111111111111
11111111111111111
11111111111111111

Your __len__ is just 1 greater than your actual data, since you loaded the df with header=None.
Simply change the last line of __init__ to self.data_len = len(self.image_arr). This should fix your problem with minimal change.
(Alternatively, set header=True while loading your df, in which case, you'll have to change iloc[1:, ...] to iloc[:, ...], as you no longer need to skip the first row.)
