title: List index out of range while training with model.fit
body: <p>First of all, I tried to look at other similar questions, but it has not fixed my issues.</p>
  
  <p>I have a project which required me to use Data Augmentation to increase the samples and Transfer Learning with VGG16 to improve the accuracy. I found this Github link online <a href="https://github.com/sachinruk/deepschool.io/blob/master/DL-Keras_Tensorflow/Lesson%2013%20-%20Transfer%20Learning%20-%20Solutions.ipynb" rel="nofollow noreferrer">https://github.com/sachinruk/deepschool.io/blob/master/DL-Keras_Tensorflow/Lesson%2013%20-%20Transfer%20Learning%20-%20Solutions.ipynb</a> and I try to apply to my project, but the error is List index out of range. I already loaded the images in data --> train or test --> benign or malign. My image's shape is (360,560,3). I am using Tensorflow 2.1.0. Here is my code.</p>
  
  <pre><code>import random
  import numpy as np
  from numpy.random import randint as rdi
  import pandas as pd
  import matplotlib.pyplot as plt
  import seaborn as sns
  
  
  from plotly import __version__
  
  import cufflinks as cf
  
  from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot
  
  init_notebook_mode(connected=True)
  cf.go_offline()
  
  import plotly.graph_objects as go
  from plotly.offline import *
  import plotly.offline as pyo
  
  import chart_studio.plotly as py
  
  from sklearn.model_selection import train_test_split
  from sklearn.linear_model import LinearRegression
  from sklearn import metrics
  from sklearn.linear_model import LogisticRegression
  from sklearn.metrics import classification_report
  from sklearn.metrics import confusion_matrix
  from sklearn.preprocessing import StandardScaler
  from sklearn.neighbors import KNeighborsClassifier
  from sklearn.tree import DecisionTreeClassifier
  from sklearn.ensemble import RandomForestClassifier
  from sklearn.datasets import load_breast_cancer
  from sklearn.svm import SVC
  from sklearn.model_selection import GridSearchCV
  from sklearn.datasets import make_blobs
  from sklearn.cluster import KMeans
  from sklearn.decomposition import PCA
  
  import tensorflow as tf
  from sklearn.preprocessing import MinMaxScaler
  from tensorflow.keras.models import Sequential
  from tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPool2D, Flatten, MaxPooling2D, Activation, Reshape, BatchNormalization
  from sklearn.metrics import mean_squared_error, mean_absolute_error, explained_variance_score
  from tensorflow.keras.models import load_model
  from tensorflow.keras.callbacks import EarlyStopping, TensorBoard
  from tensorflow.keras.datasets import mnist
  from tensorflow.keras.utils import to_categorical
  from tensorflow.keras.datasets import cifar10
  import os
  from matplotlib.image import imread
  from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
  import warnings
  from tensorflow.keras.preprocessing import image
  from mpl_toolkits.mplot3d import Axes3D
  from tensorflow.keras.optimizers import SGD
  from tensorflow.keras.layers import GaussianNoise
  
  from keras import applications
  
  import cv2
  from os.path import isfile, isdir, getsize
  from tqdm import tqdm
  from urllib.request import urlretrieve
  import zipfile
  import tarfile
  import pickle
  import glob
  import shutil
  from os.path import isfile, isdir, getsize
  from os import mkdir, makedirs, remove, listdir
  
  
  # Try to download bottelneck file for later use
  class DLProgress(tqdm):
      last_block = 0
  
      def hook(self, block_num=1, block_size=1, total_size=None):
          self.total = total_size
          self.update((block_num - self.last_block) * block_size)
          self.last_block = block_num
  
  class DLProgress(tqdm):
      last_block = 0
  
      def hook(self, block_num=1, block_size=1, total_size=None):
          self.total = total_size
          self.update((block_num - self.last_block) * block_size)
          self.last_block = block_num
  
  
  if not isfile('bottleneck_features_train.npy'):
      with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Bottleneck features') as pbar:
          urlretrieve(
                  'https://www.dropbox.com/s/a38gpvdcryw0kfc/bottleneck.zip?dl=1',
                  'bottleneck.zip',
                  pbar.hook)
  
      with zipfile.ZipFile('bottleneck.zip') as f:
          f.extractall('./')
  
      files = listdir('bottleneck 2/')
  
      for f in files:
          shutil.move('bottleneck 2/'+f,'./')
      shutil.rmtree('bottleneck 2/')
      remove('bottleneck.zip')
  
  # Import the Ultrasound dataset
  data_dir = 'C:\Spring 2020\Machine Learning and Computer Vision\data'
  os.listdir(data_dir)
  test_path = data_dir+'\\test\\'
  train_path = data_dir+'\\train\\'
  os.listdir(test_path)
  os.listdir(train_path)
  os.listdir(train_path+'\\malign')[0]
  para_cell = train_path+'\\malign'+'\\53.jpg'
  para_img= imread(para_cell)
  para_img.shape
  plt.imshow(para_img)
  
  
  image_shape = (360,560,3)
  
  # Generate some new images
  datagen = ImageDataGenerator(rescale=1.0/255)    
  
  # Create model
  model = applications.VGG16(include_top = False, input_shape = image_shape)
  model = applications.VGG16(include_top = False, weights = 'imagenet')
  model.summary()
  
  with open('bottleneck_features_train.npy','rb') as f:
      bottleneck_features_train = pickle.load(f)
  
  
  bottleneck_features_train.shape
  
  batch_size = 128
  generator = datagen.flow_from_directory(
      train_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  batch_size = 128
  valid_generator = datagen.flow_from_directory(
      test_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  # Use bottleneck file
  with open('bottleneck_features_train.npy','rb') as f:
      bottleneck_features_train = pickle.load(f)
  
  # Apply model
  model = Sequential()
  model.add(Flatten(input_shape=bottleneck_features_train.shape[1:]))
  model.add(Dense(256, activation='relu'))
  model.add(Dropout(0.5))
  model.add(Dense(1, activation='sigmoid'))
  
  model.compile(optimizer='rmsprop',
                loss='binary_crossentropy', metrics=['accuracy'])
  model.summary()
  
  batch_size = 128
  
  # Set up the flow of training data
  generator = datagen.flow_from_directory(
      train_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  generator.class_indices
  
  # Set up the early stop to avoid training so many epochs, but won't increase the accuracy
  early_stop = EarlyStopping(monitor='val_loss',patience=5)
  warnings.filterwarnings('ignore')
  
  # I think the code stucks here
  results = model.fit(bottleneck_features_train, epochs=15, batch_size = batch_size)
  
  # Save to vgg16
  model.save('vgg16.h5')
  
  losses = pd.DataFrame(model.history.history)
  losses[['loss','val_loss']].plot()
  
  model.metrics_names
  model.evaluate_generator(test_image_gen)
  
  #https://datascience.stackexchange.com/questions/13894/how-to-get-predictions-with-predict-generator-on-streaming-test-data-in-keras
  pred_probabilities = model.predict_generator(test_image_gen)
  #preds_num = pred_probabilities
  test_image_gen.classes
  predictions = pred_probabilities &gt; 0.5
  # Numpy can treat this as True/False for us
  predictions
  
  check = classification_report(test_image_gen.classes,predictions)
  print(classification_report(test_image_gen.classes,predictions))
  print(confusion_matrix(test_image_gen.classes,predictions))
  
  
  para_cell
  my_image = image.load_img(para_cell,target_size=image_shape)
  my_image
  type(my_image)
  my_image = image.img_to_array(my_image)
  type(my_image)
  my_image.shape
  my_image = np.expand_dims(my_image, axis=0)
  my_image.shape
  
  para_cell = test_path+'\\benign'+'\\40.jpg'
  my_image = image.load_img(para_cell,target_size=image_shape)
  my_image
  type(my_image)
  my_image = image.img_to_array(my_image)
  type(my_image)
  my_image.shape
  my_image = np.expand_dims(my_image, axis=0)
  my_image.shape
  model.predict(my_image)
  </code></pre>
  
  <p>The error is:</p>
  
  <pre><code>runfile('C:/Spring 2020/Machine Learning and Computer Vision/hopefully.py', wdir='C:/Spring 2020/Machine Learning and Computer Vision')
  
  
  Model: "vgg16"
  _________________________________________________________________
  Layer (type)                 Output Shape              Param #   
  =================================================================
  input_16 (InputLayer)        (None, None, None, 3)     0         
  _________________________________________________________________
  block1_conv1 (Conv2D)        (None, None, None, 64)    1792      
  _________________________________________________________________
  block1_conv2 (Conv2D)        (None, None, None, 64)    36928     
  _________________________________________________________________
  block1_pool (MaxPooling2D)   (None, None, None, 64)    0         
  _________________________________________________________________
  block2_conv1 (Conv2D)        (None, None, None, 128)   73856     
  _________________________________________________________________
  block2_conv2 (Conv2D)        (None, None, None, 128)   147584    
  _________________________________________________________________
  block2_pool (MaxPooling2D)   (None, None, None, 128)   0         
  _________________________________________________________________
  block3_conv1 (Conv2D)        (None, None, None, 256)   295168    
  _________________________________________________________________
  block3_conv2 (Conv2D)        (None, None, None, 256)   590080    
  _________________________________________________________________
  block3_conv3 (Conv2D)        (None, None, None, 256)   590080    
  _________________________________________________________________
  block3_pool (MaxPooling2D)   (None, None, None, 256)   0         
  _________________________________________________________________
  block4_conv1 (Conv2D)        (None, None, None, 512)   1180160   
  _________________________________________________________________
  block4_conv2 (Conv2D)        (None, None, None, 512)   2359808   
  _________________________________________________________________
  block4_conv3 (Conv2D)        (None, None, None, 512)   2359808   
  _________________________________________________________________
  block4_pool (MaxPooling2D)   (None, None, None, 512)   0         
  _________________________________________________________________
  block5_conv1 (Conv2D)        (None, None, None, 512)   2359808   
  _________________________________________________________________
  block5_conv2 (Conv2D)        (None, None, None, 512)   2359808   
  _________________________________________________________________
  block5_conv3 (Conv2D)        (None, None, None, 512)   2359808   
  _________________________________________________________________
  block5_pool (MaxPooling2D)   (None, None, None, 512)   0         
  =================================================================
  Total params: 14,714,688
  Trainable params: 14,714,688
  Non-trainable params: 0
  _________________________________________________________________
  Found 2198 images belonging to 2 classes.
  Found 800 images belonging to 2 classes.
  Model: "sequential_7"
  _________________________________________________________________
  Layer (type)                 Output Shape              Param #   
  =================================================================
  flatten_7 (Flatten)          (None, 4608)              0         
  _________________________________________________________________
  dense_14 (Dense)             (None, 256)               1179904   
  _________________________________________________________________
  dropout_7 (Dropout)          (None, 256)               0         
  _________________________________________________________________
  dense_15 (Dense)             (None, 1)                 257       
  =================================================================
  Total params: 1,180,161
  Trainable params: 1,180,161
  Non-trainable params: 0
  _________________________________________________________________
  Found 2198 images belonging to 2 classes.
  Train on 19872 samples
  Epoch 1/15
    128/19872 [..............................] - ETA: 2sTraceback (most recent call last):
  
    File "C:\Spring 2020\Machine Learning and Computer Vision\hopefully.py", line 188, in &lt;module&gt;
      results = model.fit(bottleneck_features_train, epochs=15, batch_size = batch_size)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training.py", line 819, in fit
      use_multiprocessing=use_multiprocessing)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py", line 342, in fit
      total_epochs=epochs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py", line 128, in run_one_epoch
      batch_outs = execution_function(iterator)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 98, in execution_function
      distributed_function(input_fn))
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 568, in __call__
      result = self._call(*args, **kwds)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 615, in _call
      self._initialize(args, kwds, add_initializers_to=initializers)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 497, in _initialize
      *args, **kwds))
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2389, in _get_concrete_function_internal_garbage_collected
      graph_function, _, _ = self._maybe_define_function(args, kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2703, in _maybe_define_function
      graph_function = self._create_graph_function(args, kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\function.py", line 2593, in _create_graph_function
      capture_by_value=self._capture_by_value),
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\framework\func_graph.py", line 978, in func_graph_from_py_func
      func_outputs = python_func(*func_args, **func_kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\eager\def_function.py", line 439, in wrapped_fn
      return weak_wrapped_fn().__wrapped__(*args, **kwds)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 85, in distributed_function
      per_replica_function, args=args)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 763, in experimental_run_v2
      return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 1819, in call_for_each_replica
      return self._call_for_each_replica(fn, args, kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\distribute\distribute_lib.py", line 2164, in _call_for_each_replica
      return fn(*args, **kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\autograph\impl\api.py", line 292, in wrapper
      return func(*args, **kwargs)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py", line 433, in train_on_batch
      output_loss_metrics=model._output_loss_metrics)
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 312, in train_on_batch
      output_loss_metrics=output_loss_metrics))
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 253, in _process_single_batch
      training=training))
  
    File "C:\Users\binhd\Anaconda3\lib\site-packages\tensorflow_core\python\keras\engine\training_eager.py", line 167, in _model_loss
      per_sample_losses = loss_fn.call(targets[i], outs[i])
  
  IndexError: list index out of range
  </code></pre>
  
  <p>Like I mentioned above, I re-used some codes from online Github to apply to my previous codes. The codes I re-used are (In just copy and paste the codes above, so you might not need to look at this part. It is just for info): </p>
  
  <pre><code># Try to download bottelneck file for later use
  class DLProgress(tqdm):
      last_block = 0
  
      def hook(self, block_num=1, block_size=1, total_size=None):
          self.total = total_size
          self.update((block_num - self.last_block) * block_size)
          self.last_block = block_num
  
  class DLProgress(tqdm):
      last_block = 0
  
      def hook(self, block_num=1, block_size=1, total_size=None):
          self.total = total_size
          self.update((block_num - self.last_block) * block_size)
          self.last_block = block_num
  
  
  if not isfile('bottleneck_features_train.npy'):
      with DLProgress(unit='B', unit_scale=True, miniters=1, desc='Bottleneck features') as pbar:
          urlretrieve(
                  'https://www.dropbox.com/s/a38gpvdcryw0kfc/bottleneck.zip?dl=1',
                  'bottleneck.zip',
                  pbar.hook)
  
      with zipfile.ZipFile('bottleneck.zip') as f:
          f.extractall('./')
  
      files = listdir('bottleneck 2/')
  
      for f in files:
          shutil.move('bottleneck 2/'+f,'./')
      shutil.rmtree('bottleneck 2/')
      remove('bottleneck.zip')
  </code></pre>
  
  <p>And (In just copy and paste the codes above, so you might not need to look at this part. It is just for info):</p>
  
  <pre><code># Generate some new images
  datagen = ImageDataGenerator(rescale=1.0/255)    
  
  # Create model
  model = applications.VGG16(include_top = False, input_shape = image_shape)
  model = applications.VGG16(include_top = False, weights = 'imagenet')
  model.summary()
  
  with open('bottleneck_features_train.npy','rb') as f:
      bottleneck_features_train = pickle.load(f)
  
  
  bottleneck_features_train.shape
  
  batch_size = 128
  generator = datagen.flow_from_directory(
      train_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  batch_size = 128
  valid_generator = datagen.flow_from_directory(
      test_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  # Use bottleneck file
  with open('bottleneck_features_train.npy','rb') as f:
      bottleneck_features_train = pickle.load(f)
  
  # Apply model
  model = Sequential()
  model.add(Flatten(input_shape=bottleneck_features_train.shape[1:]))
  model.add(Dense(256, activation='relu'))
  model.add(Dropout(0.5))
  model.add(Dense(1, activation='sigmoid'))
  
  model.compile(optimizer='rmsprop',
                loss='binary_crossentropy', metrics=['accuracy'])
  model.summary()
  
  batch_size = 128
  
  # Set up the flow of training data
  generator = datagen.flow_from_directory(
      train_path,
      target_size=image_shape[:2],
      batch_size=batch_size,
      class_mode=None,
      shuffle=False)
  
  generator.class_indices
  
  # Set up the early stop to avoid training so many epochs, but won't increase the accuracy
  early_stop = EarlyStopping(monitor='val_loss',patience=5)
  warnings.filterwarnings('ignore')
  
  # I think the code stucks here
  results = model.fit(bottleneck_features_train, epochs=15, batch_size = batch_size)
  </code></pre>
  
  <p>How can I fix this issue?</p>