title: Evaluation of ResNet50 returns numpy array of only zeroes for binary classification
body: <p>I'm trying to implement ResNet50 on ImageNet pre-trained weights. Below is the full code:</p>
  <pre><code>from tensorflow import keras
  from tensorflow.keras.models import Model
  from tensorflow.keras.layers import Dense
  from tensorflow.keras import optimizers
  from tensorflow.keras.preprocessing.image import ImageDataGenerator
  from tensorflow.keras.preprocessing import image
  
  #Data preparation below
  train_datagen = ImageDataGenerator(rescale = 1./255,
                                     shear_range = 0.2,
                                     zoom_range = 0.2,
                                     horizontal_flip = True)
  
  test_datagen = ImageDataGenerator(rescale = 1./255)
  
  traindata = train_datagen.flow_from_directory('dataset/training_set',
                                                   target_size = (50, 50),
                                                   batch_size = 32,
                                                   class_mode = 'categorical')
  testdata = test_datagen.flow_from_directory('dataset/test_set',
                                              target_size = (50, 50),
                                              batch_size = 32,
                                              class_mode = 'categorical')
  ''' Building RESNET50 by freezing convolutional layers, clipping FC layers (include_top=False) and # manually including them by Dense layers. If include_top is set to True, a custom input shape of 50x50x3 cannot be provided which is imperative due to the size of the images of dataset'''
  
  from tensorflow.keras.applications import ResNet50
  resnet50model = ResNet50(weights='imagenet', include_top=False, input_shape=(50, 50, 3))
  
  from tensorflow.keras.models import Sequential
  from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D , Flatten
  resnet50_1 = Sequential()
  resnet50_1.add(Flatten())
  resnet50_1.add(Dense(units=1000,activation=&quot;relu&quot;))
  resnet50_1.add(Dense(units=2, activation=&quot;softmax&quot;))
  resnet50model_new = Model(inputs = resnet50model.input, outputs = resnet50_1(resnet50model.output))
  #freezing convolutional layers below
  for layers in (resnet50model_new.layers)[:-2]:
      print(layers)
      layers.trainable = False
  
  #Using keras-buoy for resumable training
  
  resnet50model_new.compile(loss = &quot;categorical_crossentropy&quot;, optimizer = optimizers.SGD(lr=0.0001, momentum=0.9), metrics=[&quot;accuracy&quot;])
  from keras_buoy.models import ResumableModel
  resumable_model_resnet50 = ResumableModel(resnet50model_new, save_every_epochs = 1, custom_objects=None, to_path='dataset/saved_models/resnet50.h5')
  history_resnet50 = resumable_model_resnet50.fit_generator(traindata, steps_per_epoch=157106/32, epochs=15, validation_data=testdata, validation_steps=55504/32)
  
  #Now, to evaluate
  
  import glob
  normal_cases_dir = xxx
  idc_cases_dir = xxx
  
  normal_cases = glob.glob(normal_cases_dir+'/*.png')
  idc_cases = glob.glob(idc_cases_dir+'/*.png')
  
  test_data = []
  test_labels = []
  import cv2
  import keras
  
  for img in normal_cases:
      img = cv2.imread(str(img))
      img = cv2.resize(img, (50,50))
      if img.shape[2] ==1:
          img = np.dstack([img, img, img])
      else:
          img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
      img = img.astype(np.float32)/255.
      label = keras.utils.to_categorical(0, num_classes=2)
      test_data.append(img)
      test_labels.append(label)
                        
  for img in idc_cases:
      img = cv2.imread(str(img))
      img = cv2.resize(img, (50,50))
      if img.shape[2] ==1:
          img = np.dstack([img, img, img])
      else:
          img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
      img = img.astype(np.float32)/255.
      label = keras.utils.to_categorical(1, num_classes=2)
      test_data.append(img)
      test_labels.append(label)
      
  
  test_data = np.array(test_data)
  test_labels = np.array(test_labels)
  
  test_loss_resnet50, test_score_resnet50 = resnet50model_new.evaluate(test_data, test_labels, batch_size=32)
  print(&quot;Loss on test set: &quot;, test_loss_resnet50)
  print(&quot;Accuracy on test set: &quot;, test_score_resnet50)
  
  '''The above 2 print statement gives:
  Loss on test set:  0.5971299170691742
  Accuracy on test set:  0.7161286
  '''
  
  preds_resnet50 = resnet50model_new.predict(test_data, batch_size = 32)
  preds_resnet50 = np.argmax(preds_resnet50, axis=-1)
  
  orig_test_labels = np.argmax(test_labels, axis=-1)
  
  cm = confusion_matrix(orig_test_labels, preds_resnet50)
  print(cm)
  
  '''The above print statements give a confusion matrix:
  array([[39748,     0],
         [15756,     0]], dtype=int64)
  '''
  
  #The above confusion matrix has zero values for false positive (FP) and true positive (TP)
  #preds_resnet50 returns:
  # array([0, 0, 0, ..., 0, 0, 0], dtype=int64)
  # all zeroes ^
  </code></pre>
  <p>The problem is that the predictions are all coming to be negative (zeroes). I found this question similar: <a href="https://stackoverflow.com/questions/45900449/keras-model-predict-always-0">Keras model.predict always 0</a>
  However there is no correct answer and all they are suggesting is to rescale values even when evaluating on test set. I have done that and still the problem persists.</p>